From 793370e1dcdecf16c8efef103520f778c8fd5fa0 Mon Sep 17 00:00:00 2001 From: Leonhard Kaufmann Date: Wed, 19 Aug 2026 10:50:49 +0200 Subject: [PATCH 1/5] Vendor Brain log-parsing engine (logpai/logparser, Apache-2.0) Ports Brain's tuple-tree template-derivation core (get_frequecy_vector, tuple_generate, tupletree.find_root/up_split/down_split, output_result) into an in-memory, file-I/O-free form: no CSV/argparse/print, just a LogParser.parse(contents: list[str]) -> list[str]) entry point. The algorithm itself, including its known quirks, is kept faithful to upstream. --- .../parsers/brain/engine/LICENSE | 203 ++++++++++ .../parsers/brain/engine/README.md | 62 +++ .../parsers/brain/engine/__init__.py | 7 + .../parsers/brain/engine/core.py | 376 ++++++++++++++++++ 4 files changed, 648 insertions(+) create mode 100644 src/detectmatelibrary/parsers/brain/engine/LICENSE create mode 100644 src/detectmatelibrary/parsers/brain/engine/README.md create mode 100644 src/detectmatelibrary/parsers/brain/engine/__init__.py create mode 100644 src/detectmatelibrary/parsers/brain/engine/core.py diff --git a/src/detectmatelibrary/parsers/brain/engine/LICENSE b/src/detectmatelibrary/parsers/brain/engine/LICENSE new file mode 100644 index 00000000..d4948256 --- /dev/null +++ b/src/detectmatelibrary/parsers/brain/engine/LICENSE @@ -0,0 +1,203 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + + Copyright (C) 2018-2023 LOGPAI (https://github.com/logpai). + Copyright (C) 2023 gaiusyu + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/src/detectmatelibrary/parsers/brain/engine/README.md b/src/detectmatelibrary/parsers/brain/engine/README.md new file mode 100644 index 00000000..d49b584e --- /dev/null +++ b/src/detectmatelibrary/parsers/brain/engine/README.md @@ -0,0 +1,62 @@ +# Brain + + +Automated log analysis can facilitate failure diagnosis for developers and operators using a large volume of logs. Log parsing is a prerequisite step for automated log analysis, which parses semi-structured logs into structured logs. However, existing parsers are difficult to apply to software-intensive systems, due to their unstable parsing accuracy on various software. Although neural network-based approaches are stable, their inefficiency makes it challenging to keep up with the speed of log production.We found that a logging statement always generate the same template words, thus, the word with the most frequency in each log is more likely to be constant. However, the identical constant and variable generated from different logging statements may break this rule Inspired by this key insight, we propose a new stable log parsing approach, called Brain, which creates initial groups according to the longest common pattern. Then a bidirectional tree is used to hierarchically complement the constant words to the longest common pattern to form the complete log template efficiently. Experimental results on 16 benchmark datasets show that our approach outperforms the state-of-the-art parsers on two widely-used parsing accuracy metrics, and it only takes around 46 seconds to process one million lines of logs. + + +### Running + +Install the required enviornment: +``` +pip install -r requirements.txt +``` + +Run the following scripts to start the demo: + +``` +python demo.py +``` + +Run the following scripts to execute the benchmark: + +``` +python benchmark.py +``` + +### Docker images + +``` +1. docker pull docker.io/gaiusyu/brain:v2 +2. docker run -it --name brain gaiusyu/brain:v2 +``` + +### Benchmark + +Running the benchmark script on Loghub_2k datasets, you could obtain the following results. + +| Dataset | F1_measure | Accuracy | +|:-----------:|:----------|:---------| +| Proxifier | 1 | 1 | +| HDFS | 0.999984 | 0.9975 | +| Hadoop | 0.998749 | 0.949 | +| Spark | 0.99998 | 0.9975 | +| Zookeeper | 0.9998 | 0.9875 | +| BGL | 0.999932 | 0.986 | +| HPC | 0.997707 | 0.945 | +| Thunderbird | 0.999933 | 0.971 | +| Windows | 0.999995 | 0.997 | +| Linux | 0.999992 | 0.996 | +| Android | 0.996837 | 0.9605 | +| HealthApp | 1 | 1 | +| Apache | 1 | 1 | +| OpenSSH | 1 | 1 | +| OpenStack | 1 | 1 | +| Mac | 0.995821 | 0.942 | + + +### 🔥 Citation + +If you use the code or benchmarking results in your publication, please kindly cite the following papers. + ++ [**TSC'23**] Siyu Yu, Pinjia He, Ningjiang Chen, and Yifan Wu. [Brain: Log Parsing with Bidirectional Parallel Tree](https://ieeexplore.ieee.org/abstract/document/10109145), *IEEE Transactions on Service Computing*, 2023. ++ [**ICSE'19**] Jieming Zhu, Shilin He, Jinyang Liu, Pinjia He, Qi Xie, Zibin Zheng, Michael R. Lyu. [Tools and Benchmarks for Automated Log Parsing](https://arxiv.org/pdf/1811.03509.pdf). *International Conference on Software Engineering (ICSE)*, 2019. diff --git a/src/detectmatelibrary/parsers/brain/engine/__init__.py b/src/detectmatelibrary/parsers/brain/engine/__init__.py new file mode 100644 index 00000000..1491172c --- /dev/null +++ b/src/detectmatelibrary/parsers/brain/engine/__init__.py @@ -0,0 +1,7 @@ +# Apache License, Version 2.0 +# +# Copyright (C) 2016-2023 LOGPAI (https://github.com/logpai) +# Copyright (C) 2023 gaiusyu +# +# Based on Brain (https://github.com/logpai/logparser/tree/main/logparser/Brain) +# See LICENSE in this directory for the full license text. diff --git a/src/detectmatelibrary/parsers/brain/engine/core.py b/src/detectmatelibrary/parsers/brain/engine/core.py new file mode 100644 index 00000000..c8b1495e --- /dev/null +++ b/src/detectmatelibrary/parsers/brain/engine/core.py @@ -0,0 +1,376 @@ +# ========================================================================= +# Copyright (C) 2016-2023 LOGPAI (https://github.com/logpai). +# Copyright (C) 2023 gaiusyu +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ========================================================================= +"""Vendored core of Brain (https://github.com/logpai/logparser/tree/main/logparser/Brain). + +Ported from ``Brain.py``'s ``LogParser``/``tupletree`` classes. The file-I/O, +CSV, CLI and dataset-loading plumbing (``load_data``, ``generateresult``, +``format_log``, ``save_result``, ``__main__``) has been stripped since this +library streams logs in-memory; the tuple-tree template derivation itself +(``get_frequecy_vector``, ``tuple_generate``, ``tupletree.find_root``, +``tupletree.up_split``, ``tupletree.down_split``, ``output_result``) is kept +faithful to upstream, including its quirks (e.g. the always-false +frequency/tuple comparison in ``up_split``, preserved below with a +``type: ignore`` rather than "fixed", since fixing it would change which +templates Brain derives). +""" + +from collections import Counter + +import regex as re + +Word = str | int +FreqTuple = tuple[int, Word, int] +FreqCountPair = tuple[int, int] + +GroupLen = dict[int, list[list[str]]] +TupleVector = dict[int, list[list[FreqTuple]]] +FrequencyVector = dict[int, list[list[int]]] +SortedTupleVector = TupleVector +WordCombinations = dict[int, list[list[FreqCountPair]]] + +RootKey = FreqCountPair +RootSetDetailID = dict[RootKey, list[list[FreqTuple]]] +RootSet = dict[RootKey, list[list[FreqCountPair]]] +RootSetDetail = dict[RootKey, list[list[FreqTuple]]] + +TemplateSet = dict[tuple[str, ...], list[int]] + +_DIGIT_RE = re.compile(r"\d") + + +def get_frequecy_vector( + sentences: list[str], + rex: list[str], + delimiter: list[str], + dataset: str = "", +) -> tuple[GroupLen, TupleVector, FrequencyVector]: + """Preprocess sentences, group by length and convert each log into a + frequency vector. + + Output: + group_len: log groups based on length (each entry is [line_id, tok1, tok2, ...]) + tuple_vector: each word converted into (word_frequency, word_character, word_position) + frequency_vector: each word converted into its frequency + """ + group_len: GroupLen = {} + word_positions: dict[str, list[str]] = {} + for line_id, raw_line in enumerate(sentences): + s = raw_line + for rgex in rex: + s = re.sub(rgex, "<*>", s) + for de in delimiter: + s = re.sub(de, "", s) + if dataset == "HealthApp": + s = re.sub(":", ": ", s) + s = re.sub("=", "= ", s) + s = re.sub(r"\|", "| ", s) + if dataset == "Android": + s = re.sub(r"\(", "( ", s) + s = re.sub(r"\)", ") ", s) + if dataset == "Android": + s = re.sub(":", ": ", s) + s = re.sub("=", "= ", s) + if dataset == "HPC": + s = re.sub("=", "= ", s) + s = re.sub("-", "- ", s) + s = re.sub(":", ": ", s) + if dataset == "BGL": + s = re.sub("=", "= ", s) + s = re.sub(r"\.\.", ".. ", s) + s = re.sub(r"\(", "( ", s) + s = re.sub(r"\)", ") ", s) + if dataset == "Hadoop": + s = re.sub("_", "_ ", s) + s = re.sub(":", ": ", s) + s = re.sub("=", "= ", s) + s = re.sub(r"\(", "( ", s) + s = re.sub(r"\)", ") ", s) + if dataset == "HDFS": + s = re.sub(":", ": ", s) + if dataset == "Linux": + s = re.sub("=", "= ", s) + s = re.sub(":", ": ", s) + if dataset == "Spark": + s = re.sub(":", ": ", s) + if dataset == "Thunderbird": + s = re.sub(":", ": ", s) + s = re.sub("=", "= ", s) + if dataset == "Windows": + s = re.sub(":", ": ", s) + s = re.sub("=", "= ", s) + s = re.sub(r"\[", "[ ", s) + s = re.sub("]", "] ", s) + if dataset == "Zookeeper": + s = re.sub(":", ": ", s) + s = re.sub("=", "= ", s) + s = re.sub(",", ", ", s) + tokens = re.sub(" +", " ", s).split(" ") + tokens.insert(0, str(line_id)) + for position, token in enumerate(tokens): + word_positions.setdefault(str(position), []).append(token) + length = len(tokens) + group_len.setdefault(length, []).append(tokens) + + tuple_vector: TupleVector = {} + frequency_vector: FrequencyVector = {} + if not group_len: + return group_len, tuple_vector, frequency_vector + + max_length = max(group_len) + freq_table: dict[str, int] = {} + for position in range(max_length): + for word in word_positions[str(position)]: + key = f"{position} {word}" + freq_table[key] = freq_table.get(key, 0) + 1 + + for length, lines in group_len.items(): + for tokens in lines: + position = 0 + fre: list[FreqTuple] = [] + fre_common: list[int] = [] + for idx, token in enumerate(tokens): + if idx == 0: + continue # skip the injected line id + frequency_word = freq_table[f"{position + 1} {token}"] + fre.append((frequency_word, token, position)) + fre_common.append(frequency_word) + position += 1 + tuple_vector.setdefault(length, []).append(fre) + frequency_vector.setdefault(length, []).append(fre_common) + + return group_len, tuple_vector, frequency_vector + + +def tuple_generate( + group_len: GroupLen, + tuple_vector: TupleVector, + frequency_vector: FrequencyVector, +) -> tuple[SortedTupleVector, WordCombinations, WordCombinations]: + """Generate word combinations. + + Output: + sorted_tuple_vector: each tuple in tuple_vector, sorted by frequency (descending). + word_combinations: words with the same frequency grouped, descending by frequency. + word_combinations_reverse: same word combinations, ascending by frequency. + """ + sorted_tuple_vector: SortedTupleVector = {} + word_combinations: WordCombinations = {} + word_combinations_reverse: WordCombinations = {} + for key in group_len: + for fre in tuple_vector[key]: + sorted_fre_reverse = sorted(fre, key=lambda tup: tup[0], reverse=True) + sorted_tuple_vector.setdefault(key, []).append(sorted_fre_reverse) + for fc in frequency_vector[key]: + counted = Counter(fc) + result = counted.most_common() + sorted_result = sorted(result, key=lambda tup: tup[1], reverse=True) + sorted_fre = sorted(result, key=lambda tup: tup[0], reverse=True) + word_combinations.setdefault(key, []).append(sorted_result) + word_combinations_reverse.setdefault(key, []).append(sorted_fre) + return sorted_tuple_vector, word_combinations, word_combinations_reverse + + +class tupletree: # lowercase name kept for fidelity with upstream Brain.py + """tupletree(sorted_tuple_vector[key], word_combinations[key], + word_combinations_reverse[key], tuple_vector[key], group_len[key])""" + + def __init__( + self, + sorted_tuple_vector: list[list[FreqTuple]], + word_combinations: list[list[FreqCountPair]], + word_combinations_reverse: list[list[FreqCountPair]], + tuple_vector: list[list[FreqTuple]], + group_len: list[list[str]], + ) -> None: + self.sorted_tuple_vector = sorted_tuple_vector + self.word_combinations = word_combinations + self.word_combinations_reverse = word_combinations_reverse + self.tuple_vector = tuple_vector + self.group_len = group_len + + def find_root(self, threshold_per: int) -> tuple[RootSetDetailID, RootSet, RootSetDetail]: + root_set_detail_id: RootSetDetailID = {} + root_set_detail: RootSetDetail = {} + root_set: RootSet = {} + for i, fc in enumerate(self.word_combinations): + count = self.group_len[i] + threshold = max(fc, key=lambda tup: tup[0])[0] * threshold_per + candidate = fc[0] + m = 0 + for fc_w in fc: + if fc_w[0] >= threshold: + self.sorted_tuple_vector[i].append((int(count[0]), -1, -1)) + root_set_detail_id.setdefault(fc_w, []).append(self.sorted_tuple_vector[i]) + root_set.setdefault(fc_w, []).append(self.word_combinations_reverse[i]) + root_set_detail.setdefault(fc_w, []).append(self.tuple_vector[i]) + break + if fc_w[0] >= m: + candidate = fc_w + m = fc_w[0] + if fc_w == fc[-1]: + self.sorted_tuple_vector[i].append((int(count[0]), -1, -1)) + root_set_detail_id.setdefault(candidate, []).append(self.sorted_tuple_vector[i]) + root_set.setdefault(candidate, []).append(self.word_combinations_reverse[i]) + root_set_detail.setdefault(fc_w, []).append(self.tuple_vector[i]) + return root_set_detail_id, root_set, root_set_detail + + def up_split(self, root_set_detail: RootSetDetailID, root_set: RootSet) -> RootSetDetailID: + for key in root_set: + tree_node = root_set[key] + father_count: list[FreqCountPair] = [] + for node in tree_node: + pos = node.index(key) + for i in range(pos): + father_count.append(node[i]) + father_set = set(father_count) + for father in father_set: + if father_count.count(father) == key[0]: + continue + for i in range(len(root_set_detail[key])): + for k in range(len(root_set_detail[key][i])): + # Upstream compares an (freq, count) pair to a full + # (freq, word, position) tuple here, which is never + # equal - so this branch never fires. Kept as-is: + # "fixing" it would change which words Brain treats + # as constants and diverge from upstream templates. + if father[0] == root_set_detail[key][i][k]: # type: ignore[comparison-overlap] + entry = root_set_detail[key][i][k] + root_set_detail[key][i][k] = (entry[0], "<*>", entry[2]) + break + return root_set_detail + + def down_split( + self, + root_set_detail_id: RootSetDetailID, + threshold: int, + root_set_detail: RootSetDetail, + ) -> RootSetDetailID: + for key in root_set_detail_id: + detail_order = root_set_detail[key] + m: list[int] = [] + child: dict[int, list[Word]] = {} + variable: set[Word] = set() + first_sentence = detail_order[0] + for m_count, det in enumerate(first_sentence): + if det[0] != key[0]: + m.append(m_count) + for i in m: + for node in detail_order: + if i < len(node): + child.setdefault(i, []).append(node[i][1]) + for i in m: + result = set(child[i]) + freq = len(result) + if freq >= threshold: + variable = variable.union(result) + for i, entries in enumerate(root_set_detail_id[key]): + for j, entry in enumerate(entries): + if isinstance(entry, tuple) and entry[1] in variable: + root_set_detail_id[key][i][j] = (entry[0], "<*>", entry[2]) + return root_set_detail_id + + +def output_result(parse_result: dict[RootKey, list[list[FreqTuple]]]) -> TemplateSet: + template_set: TemplateSet = {} + for results in parse_result.values(): + for pr in results: + sorted_pr = sorted(pr, key=lambda tup: tup[2]) + template: list[str] = [] + for entry in sorted_pr[1:]: + word = entry[1] + if not isinstance(word, str): + continue # only the (already-skipped) sentinel is an int + if "<*>" in word: + template.append("<*>") + continue + if exclude_digits(word): + template.append("<*>") + continue + template.append(word) + template_set.setdefault(tuple(template), []).append(pr[-1][0]) + return template_set + + +def exclude_digits(string: str) -> bool: + """Exclude the digits-domain words from partial constant.""" + digits = _DIGIT_RE.findall(string) + if not digits: + return False + return len(digits) / len(string) >= 0.3 + + +def derive_templates( + contents: list[str], + threshold: int = 2, + delimiter: list[str] | None = None, + rex: list[str] | None = None, + dataset: str = "", +) -> list[str]: + """Run Brain's tuple-tree algorithm over a batch of log contents and return + the derived template strings (words joined by spaces, variable positions + marked with ``<*>``).""" + if not contents: + return [] + + group_len, tuple_vector, frequency_vector = get_frequecy_vector( + contents, rex or [], delimiter or [], dataset + ) + sorted_tuple_vector, word_combinations, word_combinations_reverse = tuple_generate( + group_len, tuple_vector, frequency_vector + ) + + templates: TemplateSet = {} + for key in group_len: + tree = tupletree( + sorted_tuple_vector[key], + word_combinations[key], + word_combinations_reverse[key], + tuple_vector[key], + group_len[key], + ) + root_set_detail_id, root_set, root_set_detail = tree.find_root(0) + root_set_detail_id = tree.up_split(root_set_detail_id, root_set) + parse_result = tree.down_split(root_set_detail_id, threshold, root_set_detail) + templates.update(output_result(parse_result)) + + return [" ".join(template) for template in templates] + + +class LogParser: + """In-memory port of Brain's ``LogParser`` (file I/O stripped). + + Mirrors upstream's constructor knobs (``threshold``, ``delimeter``/ + ``delimiter``, ``rex``) minus the file-path arguments, which have no + meaning for a streaming, in-memory parser. + """ + + def __init__( + self, + threshold: int = 2, + delimiter: list[str] | None = None, + rex: list[str] | None = None, + dataset: str = "", + ) -> None: + self.threshold = threshold + self.delimiter = delimiter or [] + self.rex = rex or [] + self.dataset = dataset + + def parse(self, contents: list[str]) -> list[str]: + """Derive the Brain template set for a batch of log message + contents.""" + return derive_templates(contents, self.threshold, self.delimiter, self.rex, self.dataset) From 3bb05aa4ea961c605698dc327c96d386ff3c41b4 Mon Sep 17 00:00:00 2001 From: Leonhard Kaufmann Date: Wed, 19 Aug 2026 10:50:59 +0200 Subject: [PATCH 2/5] Add BrainParser: wrap the vendored Brain engine as a CoreParser Bridges Brain's batch algorithm into the streaming CoreParser lifecycle: train() buffers each log's content (stripped of log_format headers, same as run() does for parse()); post_train() runs Brain once over the buffer to derive a template set and builds a TemplateMatcher from it; parse() then looks templates up per-log via that matcher instead of re-running Brain. No further templates are learned once training ends. --- .../parsers/brain/__init__.py | 21 +++++ src/detectmatelibrary/parsers/brain/parser.py | 87 +++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 src/detectmatelibrary/parsers/brain/__init__.py create mode 100644 src/detectmatelibrary/parsers/brain/parser.py diff --git a/src/detectmatelibrary/parsers/brain/__init__.py b/src/detectmatelibrary/parsers/brain/__init__.py new file mode 100644 index 00000000..f28fdd9c --- /dev/null +++ b/src/detectmatelibrary/parsers/brain/__init__.py @@ -0,0 +1,21 @@ +# Apache License, Version 2.0 +# +# Copyright (C) 2016-2023 LOGPAI (https://github.com/logpai) +# Copyright (C) 2023 gaiusyu +# +# Based on Brain (https://github.com/logpai/logparser/tree/main/logparser/Brain) +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa +from .parser import BrainParserConfig, BrainParser # noqa: F401 diff --git a/src/detectmatelibrary/parsers/brain/parser.py b/src/detectmatelibrary/parsers/brain/parser.py new file mode 100644 index 00000000..b2818e9c --- /dev/null +++ b/src/detectmatelibrary/parsers/brain/parser.py @@ -0,0 +1,87 @@ +from detectmatelibrary.common.parser import CoreParser, CoreParserConfig +from detectmatelibrary.parsers.brain.engine.core import LogParser as BrainCore +from detectmatelibrary.parsers.template_matcher._matcher_op import TemplateMatcher +from detectmatelibrary.utils.log_format_utils import get_format_variables +from detectmatelibrary import schemas + +from typing import Any, cast + + +class BrainParserConfig(CoreParserConfig): + """Configuration for BrainParser.""" + method_type: str = "brain_parser" + threshold: int = 2 + delimiter: list[str] = [] + rex: list[str] = [] + + +class BrainParser(CoreParser): + """Brain (https://github.com/logpai/logparser/tree/main/logparser/Brain) + wrapped as a CoreParser. + + Brain is a *batch* algorithm: it builds its word-frequency tables and + derives templates by looking at an entire log corpus at once, whereas + DetectMateLibrary parsers are *streaming* - ``parse()`` is called once + per log, with no guarantee the whole corpus is available up front. To + bridge this mismatch, Brain is not run per-log. Instead: + + * ``train()`` only buffers each log's content (no template mining yet). + * ``post_train()`` - called once by the base component when the + training phase ends - runs Brain's tuple-tree algorithm exactly once + over the buffered corpus, deriving a fixed template set, then builds + a ``TemplateMatcher`` from it and discards the buffer. + * ``parse()`` looks up the matching template for each subsequent log + via that ``TemplateMatcher``, in O(candidate templates) rather than + re-running Brain. No new templates are learned after training ends. + """ + + def __init__( + self, + name: str = "BrainParser", + config: BrainParserConfig | dict[str, Any] = BrainParserConfig(), + ) -> None: + if isinstance(config, dict): + config = BrainParserConfig.from_dict(config, name) + + super().__init__(name=name, config=config) + self.config: BrainParserConfig + + self._engine = BrainCore( + threshold=self.config.threshold, + delimiter=self.config.delimiter, + rex=self.config.rex, + ) + self._buffer: list[str] = [] + self.template_matcher: TemplateMatcher | None = None + + def train(self, input_: schemas.LogSchema) -> None: # type: ignore + config = cast(CoreParserConfig, self.config) + _, content = get_format_variables( + config._regex, + log=input_["log"], + time_format=config.time_format, + time_format_handler=self.time_format_handler, + ) + self._buffer.append(content) + + def post_train(self) -> None: + templates = self._engine.parse(self._buffer) + self.template_matcher = TemplateMatcher(template_list=templates) if templates else None + self._buffer = [] + + def parse( + self, + input_: schemas.LogSchema, + output_: schemas.ParserSchema, + ) -> None: + if self.template_matcher is None: + output_["template"] = "" + output_["variables"] = [] + output_["EventID"] = -1 + return + + parsed = self.template_matcher(input_["log"]) + + output_["template"] = parsed["EventTemplate"] + output_["variables"] = list(parsed["Params"]) + output_["EventID"] = int(parsed["EventId"]) From 79124746e6aa36be094fd9ea76ba373b1088ecd2 Mon Sep 17 00:00:00 2001 From: Leonhard Kaufmann Date: Wed, 19 Aug 2026 10:51:05 +0200 Subject: [PATCH 3/5] Add tests for BrainParser Covers config/type wiring, the pre-training fallback, the train -> post_train -> parse lifecycle end-to-end on synthetic logs, and the vendored Brain engine directly. --- tests/test_parsers/test_brain_parser.py | 101 ++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tests/test_parsers/test_brain_parser.py diff --git a/tests/test_parsers/test_brain_parser.py b/tests/test_parsers/test_brain_parser.py new file mode 100644 index 00000000..5442dc36 --- /dev/null +++ b/tests/test_parsers/test_brain_parser.py @@ -0,0 +1,101 @@ +"""Tests for BrainParser integration. + +Brain is a batch algorithm bridged into the streaming CoreParser lifecycle +via train() / post_train() / parse() - see the docstring on BrainParser for +details. These tests exercise that lifecycle end-to-end with small synthetic +logs, deterministically and without any network or filesystem access. +""" + +from detectmatelibrary.common.parser import CoreParser +from detectmatelibrary.parsers.brain import BrainParser, BrainParserConfig +from detectmatelibrary.parsers.brain.engine.core import LogParser as BrainCore +from detectmatelibrary.utils.aux import time_test_mode +from detectmatelibrary import schemas + +time_test_mode() + +TRAIN_LOGS = [ + "Connection from 192.168.1.1 port 22", + "Connection from 192.168.1.2 port 23", + "Connection from 192.168.1.3 port 24", +] +PARSE_LOG = "Connection from 192.168.1.4 port 25" + + +def _train(parser: BrainParser, logs: list[str]) -> schemas.ParserSchema: + """Feed logs through process() (which drives train -> post_train once + exhausted) and return the last result.""" + result = None + for i, log in enumerate(logs): + result = parser.process(schemas.LogSchema({"logID": str(i), "log": log})) + assert result is not None + return result + + +class TestBrainParserInit: + def test_is_core_parser(self): + parser = BrainParser(config=BrainParserConfig()) + assert isinstance(parser, CoreParser) + + def test_config_method_type(self): + config = BrainParserConfig() + assert config.method_type == "brain_parser" + + def test_no_matcher_before_training(self): + parser = BrainParser(config=BrainParserConfig()) + assert parser.template_matcher is None + + +class TestBrainParserLifecycle: + def test_parse_before_training_falls_back(self): + parser = BrainParser(config=BrainParserConfig(data_use_training=3)) + output_data = schemas.ParserSchema() + parser.parse(schemas.LogSchema({"log": PARSE_LOG}), output_data) + + assert output_data.template == "" + assert output_data.EventID == -1 + assert output_data.variables == [] + + def test_post_train_builds_matcher(self): + parser = BrainParser(config=BrainParserConfig(data_use_training=len(TRAIN_LOGS))) + # post_train() fires once the training quota is exhausted, which the + # FitLogic state machine only detects on the *next* process() call. + for i, log in enumerate(TRAIN_LOGS + [PARSE_LOG]): + parser.process(schemas.LogSchema({"logID": str(i), "log": log})) + + assert parser.template_matcher is not None + assert parser._buffer == [] + + def test_template_has_wildcard(self): + parser = BrainParser(config=BrainParserConfig(data_use_training=len(TRAIN_LOGS))) + result = _train(parser, TRAIN_LOGS + [PARSE_LOG]) + + assert "<*>" in result["template"] + + def test_variables_extracted(self): + parser = BrainParser(config=BrainParserConfig(data_use_training=len(TRAIN_LOGS))) + result = _train(parser, TRAIN_LOGS + [PARSE_LOG]) + + assert "192.168.1.4" in result["variables"] + assert "25" in result["variables"] + + def test_event_id_is_int(self): + parser = BrainParser(config=BrainParserConfig(data_use_training=len(TRAIN_LOGS))) + result = _train(parser, TRAIN_LOGS + [PARSE_LOG]) + + assert isinstance(result["EventID"], int) + assert result["EventID"] != -1 + + +class TestBrainEngine: + """The vendored Brain core, exercised directly (no CoreParser).""" + + def test_derives_expected_template(self): + engine = BrainCore(threshold=2) + templates = engine.parse(TRAIN_LOGS) + + assert templates == ["Connection from <*> port <*>"] + + def test_empty_batch_returns_no_templates(self): + engine = BrainCore(threshold=2) + assert engine.parse([]) == [] From cdf95e638efe606d3a0e2ad82ea4daf183564289 Mon Sep 17 00:00:00 2001 From: Leonhard Kaufmann Date: Wed, 19 Aug 2026 11:09:36 +0200 Subject: [PATCH 4/5] Strengthen BrainParser tests: multi-shape corpora, no-match after training Adds a training corpus with two genuinely different log shapes and asserts each gets its own template + EventID, and a detect-phase log that matches neither template, asserting the ""/EventID=-1 fallback fires via TemplateMatcher's own no-match path (matcher already built), not the earlier "training not finished yet" fallback branch. --- tests/test_parsers/test_brain_parser.py | 61 +++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/tests/test_parsers/test_brain_parser.py b/tests/test_parsers/test_brain_parser.py index 5442dc36..4e5eb17a 100644 --- a/tests/test_parsers/test_brain_parser.py +++ b/tests/test_parsers/test_brain_parser.py @@ -21,6 +21,19 @@ ] PARSE_LOG = "Connection from 192.168.1.4 port 25" +# Two distinct log shapes (different lengths -> different Brain length buckets). +MULTI_SHAPE_TRAIN_LOGS = [ + "Connection from 192.168.1.1 port 22", + "Connection from 192.168.1.2 port 23", + "Connection from 192.168.1.3 port 24", + "User root login failed", + "User admin login failed", + "User guest login failed", +] +CONNECTION_PARSE_LOG = "Connection from 192.168.1.9 port 99" +USER_PARSE_LOG = "User nobody login failed" +UNRELATED_LOG = "Completely unrelated shutdown event happened" + def _train(parser: BrainParser, logs: list[str]) -> schemas.ParserSchema: """Feed logs through process() (which drives train -> post_train once @@ -87,6 +100,54 @@ def test_event_id_is_int(self): assert result["EventID"] != -1 +class TestBrainParserMultiShape: + """Training corpora with more than one log shape (length bucket).""" + + def test_distinct_shapes_get_distinct_templates(self): + config = BrainParserConfig(data_use_training=len(MULTI_SHAPE_TRAIN_LOGS)) + parser = BrainParser(config=config) + + for i, log in enumerate(MULTI_SHAPE_TRAIN_LOGS): + parser.process(schemas.LogSchema({"logID": str(i), "log": log})) + # First process() call past the training quota both finalizes + # post_train() and parses this same log through the fresh matcher. + conn_result = parser.process( + schemas.LogSchema({"logID": "conn", "log": CONNECTION_PARSE_LOG}) + ) + user_result = parser.process( + schemas.LogSchema({"logID": "user", "log": USER_PARSE_LOG}) + ) + + assert conn_result["template"] == "Connection from <*> port <*>" + assert user_result["template"] == "User <*> login failed" + assert conn_result["template"] != user_result["template"] + + assert isinstance(conn_result["EventID"], int) + assert isinstance(user_result["EventID"], int) + assert conn_result["EventID"] != user_result["EventID"] + + assert "192.168.1.9" in conn_result["variables"] + assert "99" in conn_result["variables"] + assert "nobody" in user_result["variables"] + + def test_no_match_after_training_uses_not_found_fallback(self): + config = BrainParserConfig(data_use_training=len(MULTI_SHAPE_TRAIN_LOGS)) + parser = BrainParser(config=config) + + for i, log in enumerate(MULTI_SHAPE_TRAIN_LOGS): + parser.process(schemas.LogSchema({"logID": str(i), "log": log})) + result = parser.process(schemas.LogSchema({"logID": "x", "log": UNRELATED_LOG})) + + # Training has genuinely finished (unlike the pre-training fallback + # test above, where template_matcher is None) - the "" + # here comes from TemplateMatcher failing to match, not from the + # "no matcher yet" branch in BrainParser.parse(). + assert parser.template_matcher is not None + assert result["template"] == "" + assert result["EventID"] == -1 + assert result["variables"] == [] + + class TestBrainEngine: """The vendored Brain core, exercised directly (no CoreParser).""" From f01c2c9cdf6a1592dffcd75a0a92c21f9467537c Mon Sep 17 00:00:00 2001 From: Leonhard Kaufmann Date: Wed, 26 Aug 2026 12:52:31 +0200 Subject: [PATCH 5/5] comment benchmark specific delimiters out --- .../parsers/brain/engine/core.py | 122 +++++++++++------- 1 file changed, 72 insertions(+), 50 deletions(-) diff --git a/src/detectmatelibrary/parsers/brain/engine/core.py b/src/detectmatelibrary/parsers/brain/engine/core.py index c8b1495e..9cba8eb1 100644 --- a/src/detectmatelibrary/parsers/brain/engine/core.py +++ b/src/detectmatelibrary/parsers/brain/engine/core.py @@ -74,49 +74,57 @@ def get_frequecy_vector( s = re.sub(rgex, "<*>", s) for de in delimiter: s = re.sub(de, "", s) - if dataset == "HealthApp": - s = re.sub(":", ": ", s) - s = re.sub("=", "= ", s) - s = re.sub(r"\|", "| ", s) - if dataset == "Android": - s = re.sub(r"\(", "( ", s) - s = re.sub(r"\)", ") ", s) - if dataset == "Android": - s = re.sub(":", ": ", s) - s = re.sub("=", "= ", s) - if dataset == "HPC": - s = re.sub("=", "= ", s) - s = re.sub("-", "- ", s) - s = re.sub(":", ": ", s) - if dataset == "BGL": - s = re.sub("=", "= ", s) - s = re.sub(r"\.\.", ".. ", s) - s = re.sub(r"\(", "( ", s) - s = re.sub(r"\)", ") ", s) - if dataset == "Hadoop": - s = re.sub("_", "_ ", s) - s = re.sub(":", ": ", s) - s = re.sub("=", "= ", s) - s = re.sub(r"\(", "( ", s) - s = re.sub(r"\)", ") ", s) - if dataset == "HDFS": - s = re.sub(":", ": ", s) - if dataset == "Linux": - s = re.sub("=", "= ", s) - s = re.sub(":", ": ", s) - if dataset == "Spark": - s = re.sub(":", ": ", s) - if dataset == "Thunderbird": - s = re.sub(":", ": ", s) - s = re.sub("=", "= ", s) - if dataset == "Windows": - s = re.sub(":", ": ", s) - s = re.sub("=", "= ", s) - s = re.sub(r"\[", "[ ", s) - s = re.sub("]", "] ", s) - if dataset == "Zookeeper": - s = re.sub(":", ": ", s) - s = re.sub("=", "= ", s) + # Disabled: the following rules are hand-tuned to the specific + # benchmark datasets (HealthApp, Android, HDFS, ...). This is a form + # of overfitting to the benchmark - the parser looks artificially + # good on exactly these datasets, but its performance no longer says + # anything about how it does on new, unseen logs. For a fair + # evaluation of generalization, the preprocessing should be + # dataset-agnostic. The logic is kept for reference but commented out. + + # if dataset == "HealthApp": + # s = re.sub(":", ": ", s) + # s = re.sub("=", "= ", s) + # s = re.sub(r"\|", "| ", s) + # if dataset == "Android": + # s = re.sub(r"\(", "( ", s) + # s = re.sub(r"\)", ") ", s) + # if dataset == "Android": + # s = re.sub(":", ": ", s) + # s = re.sub("=", "= ", s) + # if dataset == "HPC": + # s = re.sub("=", "= ", s) + # s = re.sub("-", "- ", s) + # s = re.sub(":", ": ", s) + # if dataset == "BGL": + # s = re.sub("=", "= ", s) + # s = re.sub(r"\.\.", ".. ", s) + # s = re.sub(r"\(", "( ", s) + # s = re.sub(r"\)", ") ", s) + # if dataset == "Hadoop": + # s = re.sub("_", "_ ", s) + # s = re.sub(":", ": ", s) + # s = re.sub("=", "= ", s) + # s = re.sub(r"\(", "( ", s) + # s = re.sub(r"\)", ") ", s) + # if dataset == "HDFS": + # s = re.sub(":", ": ", s) + # if dataset == "Linux": + # s = re.sub("=", "= ", s) + # s = re.sub(":", ": ", s) + # if dataset == "Spark": + # s = re.sub(":", ": ", s) + # if dataset == "Thunderbird": + # s = re.sub(":", ": ", s) + # s = re.sub("=", "= ", s) + # if dataset == "Windows": + # s = re.sub(":", ": ", s) + # s = re.sub("=", "= ", s) + # s = re.sub(r"\[", "[ ", s) + # s = re.sub("]", "] ", s) + # if dataset == "Zookeeper": + # s = re.sub(":", ": ", s) + # s = re.sub("=", "= ", s) s = re.sub(",", ", ", s) tokens = re.sub(" +", " ", s).split(" ") tokens.insert(0, str(line_id)) @@ -202,7 +210,9 @@ def __init__( self.tuple_vector = tuple_vector self.group_len = group_len - def find_root(self, threshold_per: int) -> tuple[RootSetDetailID, RootSet, RootSetDetail]: + def find_root( + self, threshold_per: int + ) -> tuple[RootSetDetailID, RootSet, RootSetDetail]: root_set_detail_id: RootSetDetailID = {} root_set_detail: RootSetDetail = {} root_set: RootSet = {} @@ -214,8 +224,12 @@ def find_root(self, threshold_per: int) -> tuple[RootSetDetailID, RootSet, RootS for fc_w in fc: if fc_w[0] >= threshold: self.sorted_tuple_vector[i].append((int(count[0]), -1, -1)) - root_set_detail_id.setdefault(fc_w, []).append(self.sorted_tuple_vector[i]) - root_set.setdefault(fc_w, []).append(self.word_combinations_reverse[i]) + root_set_detail_id.setdefault(fc_w, []).append( + self.sorted_tuple_vector[i] + ) + root_set.setdefault(fc_w, []).append( + self.word_combinations_reverse[i] + ) root_set_detail.setdefault(fc_w, []).append(self.tuple_vector[i]) break if fc_w[0] >= m: @@ -223,12 +237,18 @@ def find_root(self, threshold_per: int) -> tuple[RootSetDetailID, RootSet, RootS m = fc_w[0] if fc_w == fc[-1]: self.sorted_tuple_vector[i].append((int(count[0]), -1, -1)) - root_set_detail_id.setdefault(candidate, []).append(self.sorted_tuple_vector[i]) - root_set.setdefault(candidate, []).append(self.word_combinations_reverse[i]) + root_set_detail_id.setdefault(candidate, []).append( + self.sorted_tuple_vector[i] + ) + root_set.setdefault(candidate, []).append( + self.word_combinations_reverse[i] + ) root_set_detail.setdefault(fc_w, []).append(self.tuple_vector[i]) return root_set_detail_id, root_set, root_set_detail - def up_split(self, root_set_detail: RootSetDetailID, root_set: RootSet) -> RootSetDetailID: + def up_split( + self, root_set_detail: RootSetDetailID, root_set: RootSet + ) -> RootSetDetailID: for key in root_set: tree_node = root_set[key] father_count: list[FreqCountPair] = [] @@ -373,4 +393,6 @@ def __init__( def parse(self, contents: list[str]) -> list[str]: """Derive the Brain template set for a batch of log message contents.""" - return derive_templates(contents, self.threshold, self.delimiter, self.rex, self.dataset) + return derive_templates( + contents, self.threshold, self.delimiter, self.rex, self.dataset + )