diff --git a/src/detectmatelibrary/schemas/schemas.proto b/src/detectmatelibrary/schemas/schemas.proto index 07f3c889..95459477 100644 --- a/src/detectmatelibrary/schemas/schemas.proto +++ b/src/detectmatelibrary/schemas/schemas.proto @@ -23,8 +23,8 @@ message ParserSchema { optional string logID = 8; optional string log = 9; map logFormatVariables = 10; - optional int32 receivedTimestamp = 11; - optional int32 parsedTimestamp = 12; + optional int64 receivedTimestamp = 11; + optional int64 parsedTimestamp = 12; } message DetectorSchema { @@ -32,12 +32,12 @@ message DetectorSchema { optional string detectorID = 2; optional string detectorType = 3; optional string alertID = 4; - optional int32 detectionTimestamp = 5; + optional int64 detectionTimestamp = 5; repeated string logIDs = 6; optional float score = 8; - repeated int32 extractedTimestamps = 9; + repeated int64 extractedTimestamps = 9; optional string description = 10; - optional int32 receivedTimestamp = 11; + optional int64 receivedTimestamp = 11; map alertsObtain = 12; } @@ -46,9 +46,9 @@ message AggregateSchema { repeated string detectorIDs = 2; repeated string detectorTypes = 3; repeated string alertIDs = 4; - optional int32 outputTimestamp = 5; + optional int64 outputTimestamp = 5; repeated string logIDs = 6; - repeated int32 extractedTimestamps = 9; + repeated int64 extractedTimestamps = 9; optional string description = 10; map alertsObtain = 12; } diff --git a/src/detectmatelibrary/schemas/schemas_pb2.py b/src/detectmatelibrary/schemas/schemas_pb2.py index cc300dc8..fd62de35 100644 --- a/src/detectmatelibrary/schemas/schemas_pb2.py +++ b/src/detectmatelibrary/schemas/schemas_pb2.py @@ -13,7 +13,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rschemas.proto\"\x1d\n\x06Schema\x12\x13\n\x0b__version__\x18\x01 \x01(\t\"\xb7\x01\n\tLogSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05logID\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03log\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tlogSource\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x15\n\x08hostname\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0cX__version__B\x08\n\x06_logIDB\x06\n\x04_logB\x0c\n\n_logSourceB\x0b\n\t_hostname\"\xa5\x04\n\x0cParserSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nparserType\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08parserID\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x14\n\x07\x45ventID\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x15\n\x08template\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\tvariables\x18\x06 \x03(\t\x12\x18\n\x0bparsedLogID\x18\x07 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05logID\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x10\n\x03log\x18\t \x01(\tH\x07\x88\x01\x01\x12\x41\n\x12logFormatVariables\x18\n \x03(\x0b\x32%.ParserSchema.LogFormatVariablesEntry\x12\x1e\n\x11receivedTimestamp\x18\x0b \x01(\x05H\x08\x88\x01\x01\x12\x1c\n\x0fparsedTimestamp\x18\x0c \x01(\x05H\t\x88\x01\x01\x1a\x39\n\x17LogFormatVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0cX__version__B\r\n\x0b_parserTypeB\x0b\n\t_parserIDB\n\n\x08_EventIDB\x0b\n\t_templateB\x0e\n\x0c_parsedLogIDB\x08\n\x06_logIDB\x06\n\x04_logB\x14\n\x12_receivedTimestampB\x12\n\x10_parsedTimestamp\"\x81\x04\n\x0e\x44\x65tectorSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ndetectorID\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0c\x64\x65tectorType\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x14\n\x07\x61lertID\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1f\n\x12\x64\x65tectionTimestamp\x18\x05 \x01(\x05H\x04\x88\x01\x01\x12\x0e\n\x06logIDs\x18\x06 \x03(\t\x12\x12\n\x05score\x18\x08 \x01(\x02H\x05\x88\x01\x01\x12\x1b\n\x13\x65xtractedTimestamps\x18\t \x03(\x05\x12\x18\n\x0b\x64\x65scription\x18\n \x01(\tH\x06\x88\x01\x01\x12\x1e\n\x11receivedTimestamp\x18\x0b \x01(\x05H\x07\x88\x01\x01\x12\x37\n\x0c\x61lertsObtain\x18\x0c \x03(\x0b\x32!.DetectorSchema.AlertsObtainEntry\x1a\x33\n\x11\x41lertsObtainEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0cX__version__B\r\n\x0b_detectorIDB\x0f\n\r_detectorTypeB\n\n\x08_alertIDB\x15\n\x13_detectionTimestampB\x08\n\x06_scoreB\x0e\n\x0c_descriptionB\x14\n\x12_receivedTimestamp\"\xf1\x02\n\x0f\x41ggregateSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x0b\x64\x65tectorIDs\x18\x02 \x03(\t\x12\x15\n\rdetectorTypes\x18\x03 \x03(\t\x12\x10\n\x08\x61lertIDs\x18\x04 \x03(\t\x12\x1c\n\x0foutputTimestamp\x18\x05 \x01(\x05H\x01\x88\x01\x01\x12\x0e\n\x06logIDs\x18\x06 \x03(\t\x12\x1b\n\x13\x65xtractedTimestamps\x18\t \x03(\x05\x12\x18\n\x0b\x64\x65scription\x18\n \x01(\tH\x02\x88\x01\x01\x12\x38\n\x0c\x61lertsObtain\x18\x0c \x03(\x0b\x32\".AggregateSchema.AlertsObtainEntry\x1a\x33\n\x11\x41lertsObtainEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0cX__version__B\x12\n\x10_outputTimestampB\x0e\n\x0c_descriptionb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\rschemas.proto\"\x1d\n\x06Schema\x12\x13\n\x0b__version__\x18\x01 \x01(\t\"\xb7\x01\n\tLogSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x12\n\x05logID\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x10\n\x03log\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x16\n\tlogSource\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x15\n\x08hostname\x18\x05 \x01(\tH\x04\x88\x01\x01\x42\x0e\n\x0cX__version__B\x08\n\x06_logIDB\x06\n\x04_logB\x0c\n\n_logSourceB\x0b\n\t_hostname\"\xa5\x04\n\x0cParserSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\nparserType\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x15\n\x08parserID\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x14\n\x07\x45ventID\x18\x04 \x01(\x05H\x03\x88\x01\x01\x12\x15\n\x08template\x18\x05 \x01(\tH\x04\x88\x01\x01\x12\x11\n\tvariables\x18\x06 \x03(\t\x12\x18\n\x0bparsedLogID\x18\x07 \x01(\tH\x05\x88\x01\x01\x12\x12\n\x05logID\x18\x08 \x01(\tH\x06\x88\x01\x01\x12\x10\n\x03log\x18\t \x01(\tH\x07\x88\x01\x01\x12\x41\n\x12logFormatVariables\x18\n \x03(\x0b\x32%.ParserSchema.LogFormatVariablesEntry\x12\x1e\n\x11receivedTimestamp\x18\x0b \x01(\x03H\x08\x88\x01\x01\x12\x1c\n\x0fparsedTimestamp\x18\x0c \x01(\x03H\t\x88\x01\x01\x1a\x39\n\x17LogFormatVariablesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0cX__version__B\r\n\x0b_parserTypeB\x0b\n\t_parserIDB\n\n\x08_EventIDB\x0b\n\t_templateB\x0e\n\x0c_parsedLogIDB\x08\n\x06_logIDB\x06\n\x04_logB\x14\n\x12_receivedTimestampB\x12\n\x10_parsedTimestamp\"\x81\x04\n\x0e\x44\x65tectorSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x17\n\ndetectorID\x18\x02 \x01(\tH\x01\x88\x01\x01\x12\x19\n\x0c\x64\x65tectorType\x18\x03 \x01(\tH\x02\x88\x01\x01\x12\x14\n\x07\x61lertID\x18\x04 \x01(\tH\x03\x88\x01\x01\x12\x1f\n\x12\x64\x65tectionTimestamp\x18\x05 \x01(\x03H\x04\x88\x01\x01\x12\x0e\n\x06logIDs\x18\x06 \x03(\t\x12\x12\n\x05score\x18\x08 \x01(\x02H\x05\x88\x01\x01\x12\x1b\n\x13\x65xtractedTimestamps\x18\t \x03(\x03\x12\x18\n\x0b\x64\x65scription\x18\n \x01(\tH\x06\x88\x01\x01\x12\x1e\n\x11receivedTimestamp\x18\x0b \x01(\x03H\x07\x88\x01\x01\x12\x37\n\x0c\x61lertsObtain\x18\x0c \x03(\x0b\x32!.DetectorSchema.AlertsObtainEntry\x1a\x33\n\x11\x41lertsObtainEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0cX__version__B\r\n\x0b_detectorIDB\x0f\n\r_detectorTypeB\n\n\x08_alertIDB\x15\n\x13_detectionTimestampB\x08\n\x06_scoreB\x0e\n\x0c_descriptionB\x14\n\x12_receivedTimestamp\"\xf1\x02\n\x0f\x41ggregateSchema\x12\x18\n\x0b__version__\x18\x01 \x01(\tH\x00\x88\x01\x01\x12\x13\n\x0b\x64\x65tectorIDs\x18\x02 \x03(\t\x12\x15\n\rdetectorTypes\x18\x03 \x03(\t\x12\x10\n\x08\x61lertIDs\x18\x04 \x03(\t\x12\x1c\n\x0foutputTimestamp\x18\x05 \x01(\x03H\x01\x88\x01\x01\x12\x0e\n\x06logIDs\x18\x06 \x03(\t\x12\x1b\n\x13\x65xtractedTimestamps\x18\t \x03(\x03\x12\x18\n\x0b\x64\x65scription\x18\n \x01(\tH\x02\x88\x01\x01\x12\x38\n\x0c\x61lertsObtain\x18\x0c \x03(\x0b\x32\".AggregateSchema.AlertsObtainEntry\x1a\x33\n\x11\x41lertsObtainEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\x0e\n\x0cX__version__B\x12\n\x10_outputTimestampB\x0e\n\x0c_descriptionb\x06proto3') _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'schemas_pb2', globals()) diff --git a/src/detectmatelibrary/utils/time_format_handler.py b/src/detectmatelibrary/utils/time_format_handler.py index 66ed7487..acac57c9 100644 --- a/src/detectmatelibrary/utils/time_format_handler.py +++ b/src/detectmatelibrary/utils/time_format_handler.py @@ -85,21 +85,15 @@ def parse_timestamp(self, time_str: str, time_format: str | None = None) -> str: if ts is not None: return ts - # 3) Numeric epoch (seconds or milliseconds) + # 3) Numeric epoch (seconds, milliseconds, microseconds or nanoseconds) if re.fullmatch(r"\d+(?:\.\d+)?", time_str): try: - if "." in time_str: - val = float(time_str) - # if value looks like milliseconds (very large), normalize - if val > 1e12: - val /= 1000.0 - return str(int(val)) - else: - ival = int(time_str) - # heuristic: length >= 13 -> treat as milliseconds - if len(time_str) >= 13: - ival //= 1000 - return str(ival) + val = int(time_str.split(".", 1)[0]) + # Fold sub-second units until the value is a plausible seconds epoch. + # 100_000_000_000 seconds is the year 5138, so anything above that is ms/us/ns. + while val > 100_000_000_000: + val //= 1000 + return str(val) except (ValueError, OverflowError): pass diff --git a/tests/test_common/test_extract_timestamp.py b/tests/test_common/test_extract_timestamp.py index 77196e54..304a3401 100644 --- a/tests/test_common/test_extract_timestamp.py +++ b/tests/test_common/test_extract_timestamp.py @@ -23,6 +23,11 @@ def test_various_time_formats(self) -> None: ("0", 0), ("1772812294", 1772812294), ("1772812294.5", 1772812294), + # Sub-second epochs: ms, us and ns all fold down to seconds + ("1772812294000", 1772812294), + ("1772812294000000", 1772812294), + ("1772812294000000000", 1772812294), + ("1772812294000.5", 1772812294), # Apache/nginx format ("04/Mar/2026:14:18:00 +0000", EXPECTED_UTC), ("04/Mar/2026:14:18:00", EXPECTED_UTC), @@ -50,3 +55,27 @@ def test_various_time_formats(self) -> None: assert result == [expected], ( f"Format '{time_str}': expected [{expected}], got {result}" ) + + def test_microsecond_epoch_fits_the_schema(self) -> None: + """Issue #271: a microsecond epoch folded only once landed in + milliseconds and overflowed the int32 timestamp fields.""" + schema = schemas.DetectorSchema({ + "extractedTimestamps": _extract_timestamp( + schemas.ParserSchema({"logFormatVariables": {"Time": "1643114452000000"}}) + ), + }) + assert str(schema) # rebuilds the protobuf -- used to raise ValueError + assert schema["extractedTimestamps"] == [1643114452] + + def test_timestamp_fields_hold_more_than_int32(self) -> None: + """Timestamps are int64, so they survive 2038 (and a stray ms + value).""" + for schema, field, value in [ + (schemas.ParserSchema(), "receivedTimestamp", 2**31), + (schemas.ParserSchema(), "parsedTimestamp", 2**31), + (schemas.DetectorSchema(), "detectionTimestamp", 2**31), + (schemas.DetectorSchema(), "receivedTimestamp", 1643114452000), + (schemas.AggregateSchema(), "outputTimestamp", 2**31), + ]: + schema[field] = value + assert getattr(schema.get_schema(), field) == value