From 65016e3d506b269d35b17dfa974d9e501c0feba5 Mon Sep 17 00:00:00 2001 From: Philipp Huth Date: Fri, 4 Dec 2020 20:21:03 +0100 Subject: [PATCH 1/6] Pcap prototype init --- can/io/pcap.py | 251 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 251 insertions(+) create mode 100644 can/io/pcap.py diff --git a/can/io/pcap.py b/can/io/pcap.py new file mode 100644 index 000000000..c95d77f8f --- /dev/null +++ b/can/io/pcap.py @@ -0,0 +1,251 @@ +""" +This module works with CAN data in Pcap log files (*.pcap). +The specification from the binary logging file is taken from +https://wiki.wireshark.org/Development/LibpcapFileFormat + +The Pcap log file has a global header, containing some global information. +Followed by zero or more records for each captured packet. +Each record consists of a packet header and a packet data segement. +|-------------|-------------|-----------|-------------|-----------|---| +|Global Header|Packet Header|Packet Data|Packet Header|Packet Data|...| +|-------------|-------------|-----------|-------------|-----------|---| + +""" + +import struct +import logging +from enum import Enum + +import gzip +from typing import Optional, cast + +import can +import can.typechecking +from can.message import Message +from can.listener import Listener +from .generic import BaseIOHandler + + +class PcapParseError(Exception): + """Pcap file could not be parsed correctly.""" + +LOG = logging.getLogger(__name__) + +GLOBAL_HEADER_FORMAT = 'IHHiIII' +GLOBAL_HEADER_STRUCT = struct.Struct(GLOBAL_HEADER_FORMAT) +PACKET_HEADER_FORMAT = 'IIII' +PACKET_HEADER_STRUCT = struct.Struct(PACKET_HEADER_FORMAT) + +CAN_MSG_EXT = 0x80000000 +CAN_ERR_FLAG = 0x20000000 +CAN_ERR_BUSERROR = 0x00000080 +CAN_ERR_DLC = 8 + +class Precision(Enum): + ms = 1 + ns = 2 + +class PcapGlobalHeader(): + def __init__(self, data, endian): + unpacked = struct.unpack(endian + GLOBAL_HEADER_STRUCT, data) # FIXME: We need the endian here + self.magic_number = unpacked[0] # magic number + self.version_major = unpacked[1] # major version number + self.version_minor = unpacked[2] # minor version number + self.thiszone = unpacked[3] # GMT to local correction + self.sigfigs = unpacked[4] # accuracy of timestamps + self.snaplen = unpacked[5] # max length of captured packets, in octets + self.network = unpacked[6] # data link type + +class PcapPacket(): + def __init__(self, header_data, endian): + unpacked = struct.unpack(endian + PCAP_PACKET_HEADER_FORMAT, header_data) + self.ts_sec = unpacked[0] + self.ts_usec = unpacked[1] + self.incl_len = unpacked[2] + self.orig_len = unpacked[3] + + + + +class PcapLogReader(BaseIOHandler): + """ + Iterator over CAN messages from a .log Logging File (candump -L). + + .. note:: + .log-format looks for example like this: + + ``(0.0) vcan0 001#8d00100100820100`` + """ + + def __init__(self, file): + """ + :param file: a path-like object or as file-like object to read from + If this is a file-like object, is has to opened in text + read mode, not binary read mode. + """ + #super().__init__(file, mode="rb") + mode="rb" + if file is None or (hasattr(file, "read") and hasattr(file, "write")): + # file is None or some file-like object + self.file = cast(Optional[can.typechecking.FileLike], file) + else: + # file is some path-like object + # FIXME: Use propper .gz detection + try: + self.file = gzip.open(cast(can.typechecking.StringPathLike, file), mode) + except: + self.file = open(cast(can.typechecking.StringPathLike, file), mode) + + global_header_data = self.file.read(GLOBAL_HEADER_STRUCT.size) + + # Read the first 4 bytes the original format to get the writers magic number + writer_magic_number = global_header_data[0:4] + if writer_magic_number == b"\xa1\xb2\xc3\xd4": # big endian + # TODO: Looks ugly + self.endian = ">" + # TODO: Call this timestamp_precision and use an enum for the possible values + self.timestamp_precision = Precision.ms + elif writer_magic_number == b"\xd4\xc3\xb2\xa1": # little endian + self.endian = "<" + self.timestamp_precision = Precision.ms + elif writer_magic_number == b"\xa1\xb2\x3c\x4d": # big endian, nanosecond-precision + self.endian = ">" + self.timestamp_precision = Precision.ms + elif writer_magic_number == b"\x4d\x3c\xb2\xa1": # little endian, nanosecond-precision + self.endian = "<" + self.timestamp_precision = Precision.ms + else: + raise PcapParseError("Unexpected endian / precision definition") + + global_header = PcapGlobalHeader(global_header_data, self.endian) + + # We only use socketcan log files + # DLT_CAN_SOCKETCAN = 227 + # https://www.tcpdump.org/linktypes.html + if global_header.network != 227: + raise PcapParseError("Unsupported link type") + + # TODO: Check against snaplen for normal and extended DLC + + super().__init__() + + # TODO: If linktype is not Socketcan (227) throw error + # https://github.com/JarryShaw/PyPCAPKit/blob/b2e22423981209662e4c42608fa80666eed2bb2e/pcapkit/const/reg/linktype.py#L236 + + def __iter__(self): + for line in self.file: + + # skip empty lines + temp = line.strip() + if not temp: + continue + + timestamp, channel, frame = temp.split() + timestamp = float(timestamp[1:-1]) + canId, data = frame.split("#") + if channel.isdigit(): + channel = int(channel) + + isExtended = len(canId) > 3 + canId = int(canId, 16) + + if data and data[0].lower() == "r": + isRemoteFrame = True + + if len(data) > 1: + dlc = int(data[1:]) + else: + dlc = 0 + + dataBin = None + else: + isRemoteFrame = False + + dlc = len(data) // 2 + dataBin = bytearray() + for i in range(0, len(data), 2): + dataBin.append(int(data[i : (i + 2)], 16)) + + if canId & CAN_ERR_FLAG and canId & CAN_ERR_BUSERROR: + msg = Message(timestamp=timestamp, is_error_frame=True) + else: + msg = Message( + timestamp=timestamp, + arbitration_id=canId & 0x1FFFFFFF, + is_extended_id=isExtended, + is_remote_frame=isRemoteFrame, + dlc=dlc, + data=dataBin, + channel=channel, + ) + yield msg + + self.stop() + + +# class CanutilsLogWriter(BaseIOHandler, Listener): +# """Logs CAN data to an ASCII log file (.log). +# This class is is compatible with "candump -L". + +# If a message has a timestamp smaller than the previous one (or 0 or None), +# it gets assigned the timestamp that was written for the last message. +# It the first message does not have a timestamp, it is set to zero. +# """ + +# def __init__(self, file, channel="vcan0", append=False): +# """ +# :param file: a path-like object or as file-like object to write to +# If this is a file-like object, is has to opened in text +# write mode, not binary write mode. +# :param channel: a default channel to use when the message does not +# have a channel set +# :param bool append: if set to `True` messages are appended to +# the file, else the file is truncated +# """ +# mode = "a" if append else "w" +# super().__init__(file, mode=mode) + +# self.channel = channel +# self.last_timestamp = None + +# def on_message_received(self, msg): +# # this is the case for the very first message: +# if self.last_timestamp is None: +# self.last_timestamp = msg.timestamp or 0.0 + +# # figure out the correct timestamp +# if msg.timestamp is None or msg.timestamp < self.last_timestamp: +# timestamp = self.last_timestamp +# else: +# timestamp = msg.timestamp + +# channel = msg.channel if msg.channel is not None else self.channel + +# if msg.is_error_frame: +# self.file.write( +# "(%f) %s %08X#0000000000000000\n" +# % (timestamp, channel, CAN_ERR_FLAG | CAN_ERR_BUSERROR) +# ) + +# elif msg.is_remote_frame: +# if msg.is_extended_id: +# self.file.write( +# "(%f) %s %08X#R\n" % (timestamp, channel, msg.arbitration_id) +# ) +# else: +# self.file.write( +# "(%f) %s %03X#R\n" % (timestamp, channel, msg.arbitration_id) +# ) + +# else: +# data = ["{:02X}".format(byte) for byte in msg.data] +# if msg.is_extended_id: +# self.file.write( +# "(%f) %s %08X#%s\n" +# % (timestamp, channel, msg.arbitration_id, "".join(data)) +# ) +# else: +# self.file.write( +# "(%f) %s %03X#%s\n" +# % (timestamp, channel, msg.arbitration_id, "".join(data)) +# ) From 152d2204deaccf0f1c637969c29bf820ef4a3e21 Mon Sep 17 00:00:00 2001 From: Philipp Huth Date: Wed, 16 Dec 2020 15:52:09 +0100 Subject: [PATCH 2/6] Add PCAP reader --- can/io/__init__.py | 1 + can/io/pcap.py | 334 +++++++++++++++++++++++++-------------------- can/io/player.py | 3 + 3 files changed, 191 insertions(+), 147 deletions(-) diff --git a/can/io/__init__.py b/can/io/__init__.py index 0d3741b05..c71ca9df4 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -13,4 +13,5 @@ from .canutils import CanutilsLogReader, CanutilsLogWriter from .csv import CSVWriter, CSVReader from .sqlite import SqliteReader, SqliteWriter +from .pcap import PcapLogReader from .printer import Printer diff --git a/can/io/pcap.py b/can/io/pcap.py index c95d77f8f..e5acb0353 100644 --- a/can/io/pcap.py +++ b/can/io/pcap.py @@ -9,7 +9,8 @@ |-------------|-------------|-----------|-------------|-----------|---| |Global Header|Packet Header|Packet Data|Packet Header|Packet Data|...| |-------------|-------------|-----------|-------------|-----------|---| - +The packet header can be a concatination of multiple 'pseudo' headers +(see 'linux cooked' header for this). """ import struct @@ -17,12 +18,14 @@ from enum import Enum import gzip -from typing import Optional, cast +from typing import cast, Optional import can +#from can.interfaces.socketcan.constants import * import can.typechecking from can.message import Message from can.listener import Listener +from can.util import dlc2len from .generic import BaseIOHandler @@ -31,15 +34,85 @@ class PcapParseError(Exception): LOG = logging.getLogger(__name__) +# Generic PCAP file headers GLOBAL_HEADER_FORMAT = 'IHHiIII' GLOBAL_HEADER_STRUCT = struct.Struct(GLOBAL_HEADER_FORMAT) PACKET_HEADER_FORMAT = 'IIII' PACKET_HEADER_STRUCT = struct.Struct(PACKET_HEADER_FORMAT) -CAN_MSG_EXT = 0x80000000 -CAN_ERR_FLAG = 0x20000000 -CAN_ERR_BUSERROR = 0x00000080 -CAN_ERR_DLC = 8 + +## Package headers +# LINKTYPE_CAN_SOCKETCAN / DLT_CAN_SOCKETCAN -> 227 +# LINKTYPE_LINUX_SLL / DLT_LINUX_SLL -> 113 +# LINKTYPE_LINUX_SLL2 / DLT_LINUX_SLL2 -> 276 +# https://www.tcpdump.org/linktypes.html + +# Linux cooked sockets. +DLT_LINUX_SLL = 113 + +# Pseudo-header as supplied by Linux SocketCAN +DLT_CAN_SOCKETCAN = 227 + +# Linux cooked sockets v2. +DLT_LINUX_SLL2 = 276 + + +# SLL struct +# See: https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL.html +SLL_HEADER_FORMAT = '!HHHQH' +SLL_HEADER_STRUCT = struct.Struct(SLL_HEADER_FORMAT) + +# SLL v2 struct +# See: https://www.tcpdump.org/linktypes/LINKTYPE_LINUX_SLL2.html +SLL2_HEADER_FORMAT = '!HHIHBBQ' +SLL2_HEADER_STRUCT = struct.Struct(SLL2_HEADER_FORMAT) + + +# Socketcan header +SOCKETCAN_CAN_ID_FORMAT = "I" +SOCKETCAN_CAN_ID_STRUCT = struct.Struct(SOCKETCAN_CAN_ID_FORMAT) +SOCKETCAN_META_FORMAT = "BBBB" +SOCKETCAN_META_STRUCT = struct.Struct(SOCKETCAN_META_FORMAT) +SOCKETCAN_HEADER_FORMAT = SOCKETCAN_CAN_ID_FORMAT + SOCKETCAN_META_FORMAT +SOCKETCAN_HEADER_STRUCT = struct.Struct(SOCKETCAN_HEADER_FORMAT) + + +# TODO: Use as many from the internal socketcan config.py as possible +CAN_EFF_FLAG = 0x80000000 # EFF/SFF is set in the MSB (extended CAN frame) +CAN_RTR_FLAG = 0x40000000 # remote transmission request +CAN_ERR_FLAG = 0x20000000 # error frame + +CAN_FLAG_MASK = CAN_EFF_FLAG | CAN_RTR_FLAG | CAN_ERR_FLAG + +CAN_EFF_MASK = 0x1FFFFFFF # extended frame format (EFF) has a 29 bit identifier +CAN_SFF_MASK = 0x000007FF # standard frame format (SFF) has a 11 bit identifier + +# error class (mask) in can_id +# NOTE: All are unsigned +CAN_ERR_TX_TIMEOUT = 0x00000001 # TX timeout (by netdevice driver) +CAN_ERR_LOSTARB = 0x00000002 # lost arbitration / data[0] +CAN_ERR_CTRL = 0x00000004 # controller problems / data[1] +CAN_ERR_PROT = 0x00000008 # protocol violations / data[2..3] +CAN_ERR_TRX = 0x00000010 # transceiver status / data[4] +CAN_ERR_ACK = 0x00000020 # received no ACK on transmission +CAN_ERR_BUSOFF = 0x00000040 # bus off +CAN_ERR_BUSERROR = 0x00000080 # bus error (may flood!) +CAN_ERR_RESTARTED = 0x00000100 # controller restarted +CAN_ERR_RESERVED = 0x1FFFFE00 # reserved bits + +# CAN FD specific filters +CANFD_BRS = 0x01 # bit rate switch (second bitrate for payload data) +CANFD_ESI = 0x02 # error state indicator of the transmitting node + + +# The fake link-layer header of Linux cooked packets. +LINUX_SLL_PROTOCOL_OFFSET = 14 # protocol +LINUX_SLL_LEN = 16 # length of the header + +# The protocols we have to check for. +LINUX_SLL_P_CAN = 0x000C # Controller Area Network +LINUX_SLL_P_CANFD = 0x000D # Controller Area Network flexible data rate + class Precision(Enum): ms = 1 @@ -47,7 +120,7 @@ class Precision(Enum): class PcapGlobalHeader(): def __init__(self, data, endian): - unpacked = struct.unpack(endian + GLOBAL_HEADER_STRUCT, data) # FIXME: We need the endian here + unpacked = struct.unpack(endian + GLOBAL_HEADER_FORMAT, data) self.magic_number = unpacked[0] # magic number self.version_major = unpacked[1] # major version number self.version_minor = unpacked[2] # minor version number @@ -58,27 +131,24 @@ def __init__(self, data, endian): class PcapPacket(): def __init__(self, header_data, endian): - unpacked = struct.unpack(endian + PCAP_PACKET_HEADER_FORMAT, header_data) + unpacked = struct.unpack(endian + PACKET_HEADER_FORMAT, header_data) self.ts_sec = unpacked[0] self.ts_usec = unpacked[1] self.incl_len = unpacked[2] self.orig_len = unpacked[3] - - class PcapLogReader(BaseIOHandler): """ - Iterator over CAN messages from a .log Logging File (candump -L). - - .. note:: - .log-format looks for example like this: - - ``(0.0) vcan0 001#8d00100100820100`` + Iterator over CAN messages from a .pcap logging file (tcpdump -i vcan0). + Only CAN messages with their corresponding linktypes are supported. Other link types are + silently ignored. """ def __init__(self, file): """ + Read and parse the generic pcap file + :param file: a path-like object or as file-like object to read from If this is a file-like object, is has to opened in text read mode, not binary read mode. @@ -86,166 +156,136 @@ def __init__(self, file): #super().__init__(file, mode="rb") mode="rb" if file is None or (hasattr(file, "read") and hasattr(file, "write")): - # file is None or some file-like object + # file parameter is None or a file(-like object) self.file = cast(Optional[can.typechecking.FileLike], file) else: - # file is some path-like object + # file parameter is a path(-like object) # FIXME: Use propper .gz detection try: + # FIXME: Two read lines for the global header is ugly + # NOTE: Only with a read we can see if it is a gzip self.file = gzip.open(cast(can.typechecking.StringPathLike, file), mode) + global_header_data = self.file.read(GLOBAL_HEADER_STRUCT.size) except: self.file = open(cast(can.typechecking.StringPathLike, file), mode) + global_header_data = self.file.read(GLOBAL_HEADER_STRUCT.size) + #global_header_data = self.file.read(GLOBAL_HEADER_STRUCT.size) - global_header_data = self.file.read(GLOBAL_HEADER_STRUCT.size) - - # Read the first 4 bytes the original format to get the writers magic number + # Read the first 4 bytes without specific encoding to get the writers magic number writer_magic_number = global_header_data[0:4] if writer_magic_number == b"\xa1\xb2\xc3\xd4": # big endian - # TODO: Looks ugly + # TODO: Change endian to a type, looks ugly self.endian = ">" - # TODO: Call this timestamp_precision and use an enum for the possible values self.timestamp_precision = Precision.ms elif writer_magic_number == b"\xd4\xc3\xb2\xa1": # little endian self.endian = "<" self.timestamp_precision = Precision.ms elif writer_magic_number == b"\xa1\xb2\x3c\x4d": # big endian, nanosecond-precision self.endian = ">" - self.timestamp_precision = Precision.ms + self.timestamp_precision = Precision.ns elif writer_magic_number == b"\x4d\x3c\xb2\xa1": # little endian, nanosecond-precision self.endian = "<" - self.timestamp_precision = Precision.ms + self.timestamp_precision = Precision.ns else: raise PcapParseError("Unexpected endian / precision definition") - global_header = PcapGlobalHeader(global_header_data, self.endian) + self.global_header = PcapGlobalHeader(global_header_data, self.endian) - # We only use socketcan log files - # DLT_CAN_SOCKETCAN = 227 - # https://www.tcpdump.org/linktypes.html - if global_header.network != 227: + # We can only use socketcan packages with generic socketcan and linux-cooked headers + if self.global_header.network != DLT_CAN_SOCKETCAN and self.global_header.network != DLT_LINUX_SLL and self.global_header.network != DLT_LINUX_SLL2: raise PcapParseError("Unsupported link type") - # TODO: Check against snaplen for normal and extended DLC - - super().__init__() - - # TODO: If linktype is not Socketcan (227) throw error - # https://github.com/JarryShaw/PyPCAPKit/blob/b2e22423981209662e4c42608fa80666eed2bb2e/pcapkit/const/reg/linktype.py#L236 def __iter__(self): - for line in self.file: - - # skip empty lines - temp = line.strip() - if not temp: - continue - - timestamp, channel, frame = temp.split() - timestamp = float(timestamp[1:-1]) - canId, data = frame.split("#") - if channel.isdigit(): - channel = int(channel) - - isExtended = len(canId) > 3 - canId = int(canId, 16) - - if data and data[0].lower() == "r": - isRemoteFrame = True - - if len(data) > 1: - dlc = int(data[1:]) - else: - dlc = 0 - - dataBin = None - else: - isRemoteFrame = False - - dlc = len(data) // 2 - dataBin = bytearray() - for i in range(0, len(data), 2): - dataBin.append(int(data[i : (i + 2)], 16)) - - if canId & CAN_ERR_FLAG and canId & CAN_ERR_BUSERROR: - msg = Message(timestamp=timestamp, is_error_frame=True) + """ + Iterate and parse all the packets int the PCAP file + """ + while True: + + packet_header_data = self.file.read(PACKET_HEADER_STRUCT.size) + if not packet_header_data: + # EOF + break + packet_header = PcapPacket(packet_header_data, self.endian) + + # Stays `0` when there is no sll header + linux_sll_size = 0 + + # Test against an SLL header + if self.global_header.network == DLT_LINUX_SLL: + sll_header_data = self.file.read(SLL_HEADER_STRUCT.size) + linux_sll_size = SLL_HEADER_STRUCT.size + _, _, _, _, protocol_type_field = SLL_HEADER_STRUCT.unpack( + sll_header_data + ) + if protocol_type_field != LINUX_SLL_P_CAN and protocol_type_field != LINUX_SLL_P_CANFD: + raise PcapParseError("Unsupported SLL protocol type") + elif self.global_header.network == DLT_LINUX_SLL2: + sll2_header_data = self.file.read(SLL2_HEADER_STRUCT.size) + protocol_type_field, _, _, _, _, _, _, _, _ = SLL2_HEADER_STRUCT.unpack( + sll2_header_data + ) + linux_sll_size = SLL2_HEADER_STRUCT.size + if protocol_type_field != LINUX_SLL_P_CAN and protocol_type_field != LINUX_SLL_P_CANFD: + raise PcapParseError("Unsupported SLLv2 protocol type") + + # NOTE: Read the header of the CAN frame (8 byte) + packet_data_frame_header = self.file.read(SOCKETCAN_HEADER_STRUCT.size) + + can_id_encoding = '>' + if self.global_header.network == DLT_LINUX_SLL or self.global_header.network == DLT_LINUX_SLL2: + # CAN-ID encoding for packets with SLL header is host encoding + # https://github.com/the-tcpdump-group/libpcap/issues/699#issuecomment-383830002 + can_id_encoding = self.endian + + can_id = struct.unpack( + can_id_encoding + SOCKETCAN_CAN_ID_FORMAT, + packet_data_frame_header[0:SOCKETCAN_CAN_ID_STRUCT.size] + )[0] + can_dlc, reserved0, reserved1, reserved3 = struct.unpack( + '>' + SOCKETCAN_META_FORMAT, + packet_data_frame_header[SOCKETCAN_CAN_ID_STRUCT.size:SOCKETCAN_HEADER_STRUCT.size] + ) + + # NOTE Data segment is full pcap data segment minus CAN header and sll segment + packet_data_frame_length = packet_header.incl_len - (SOCKETCAN_HEADER_STRUCT.size + linux_sll_size) + # can_dlc - 8 bit CAN header should be remaining + if packet_data_frame_length == (can_dlc - SOCKETCAN_HEADER_STRUCT.size): + raise PcapParseError("Data frame size not equal to CAN frame DLC") + + packet_data_frame = self.file.read(packet_data_frame_length) + + timestamp = packet_header.ts_sec + ( packet_header.ts_usec / 1000000 ) + + # NOTE: Standard Pcap file do not store the interface name + # TODO: Add PCAPng to store multi channel + channel = 0 + + if can_id & CAN_ERR_FLAG and can_id & CAN_ERR_BUSERROR: + msg = Message( + timestamp = timestamp, + is_error_frame = True, + is_extended_id = bool(can_id & CAN_EFF_FLAG), + arbitration_id = can_id & CAN_EFF_MASK, + dlc = dlc2len(can_dlc), + data = packet_data_frame, + channel = channel, + ) else: msg = Message( - timestamp=timestamp, - arbitration_id=canId & 0x1FFFFFFF, - is_extended_id=isExtended, - is_remote_frame=isRemoteFrame, - dlc=dlc, - data=dataBin, - channel=channel, + timestamp = timestamp, + arbitration_id = can_id & CAN_EFF_MASK, + is_extended_id = bool(can_id & CAN_EFF_FLAG), + is_remote_frame = bool(can_id & CAN_RTR_FLAG), + # TODO: Use `LINUX_SLL_P_CANFD` when we can not get it from the CAN frame + is_fd = bool(reserved0 & CANFD_BRS), + bitrate_switch = bool(reserved0 & CANFD_BRS), + error_state_indicator = bool(can_id & CAN_ERR_FLAG), + dlc = dlc2len(can_dlc), + data = packet_data_frame, + channel = channel, ) - yield msg + yield msg + # END OF CAN message loop self.stop() - - -# class CanutilsLogWriter(BaseIOHandler, Listener): -# """Logs CAN data to an ASCII log file (.log). -# This class is is compatible with "candump -L". - -# If a message has a timestamp smaller than the previous one (or 0 or None), -# it gets assigned the timestamp that was written for the last message. -# It the first message does not have a timestamp, it is set to zero. -# """ - -# def __init__(self, file, channel="vcan0", append=False): -# """ -# :param file: a path-like object or as file-like object to write to -# If this is a file-like object, is has to opened in text -# write mode, not binary write mode. -# :param channel: a default channel to use when the message does not -# have a channel set -# :param bool append: if set to `True` messages are appended to -# the file, else the file is truncated -# """ -# mode = "a" if append else "w" -# super().__init__(file, mode=mode) - -# self.channel = channel -# self.last_timestamp = None - -# def on_message_received(self, msg): -# # this is the case for the very first message: -# if self.last_timestamp is None: -# self.last_timestamp = msg.timestamp or 0.0 - -# # figure out the correct timestamp -# if msg.timestamp is None or msg.timestamp < self.last_timestamp: -# timestamp = self.last_timestamp -# else: -# timestamp = msg.timestamp - -# channel = msg.channel if msg.channel is not None else self.channel - -# if msg.is_error_frame: -# self.file.write( -# "(%f) %s %08X#0000000000000000\n" -# % (timestamp, channel, CAN_ERR_FLAG | CAN_ERR_BUSERROR) -# ) - -# elif msg.is_remote_frame: -# if msg.is_extended_id: -# self.file.write( -# "(%f) %s %08X#R\n" % (timestamp, channel, msg.arbitration_id) -# ) -# else: -# self.file.write( -# "(%f) %s %03X#R\n" % (timestamp, channel, msg.arbitration_id) -# ) - -# else: -# data = ["{:02X}".format(byte) for byte in msg.data] -# if msg.is_extended_id: -# self.file.write( -# "(%f) %s %08X#%s\n" -# % (timestamp, channel, msg.arbitration_id, "".join(data)) -# ) -# else: -# self.file.write( -# "(%f) %s %03X#%s\n" -# % (timestamp, channel, msg.arbitration_id, "".join(data)) -# ) diff --git a/can/io/player.py b/can/io/player.py index 21e716250..99b2f2da1 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -19,6 +19,7 @@ from .canutils import CanutilsLogReader from .csv import CSVReader from .sqlite import SqliteReader +from .pcap import PcapLogReader class LogReader(BaseIOHandler): @@ -31,6 +32,7 @@ class LogReader(BaseIOHandler): * .csv * .db * .log + * .pcap Exposes a simple iterator interface, to use simply: @@ -53,6 +55,7 @@ class LogReader(BaseIOHandler): ".csv": CSVReader, ".db": SqliteReader, ".log": CanutilsLogReader, + ".pcap": PcapLogReader, } @staticmethod From 77bee0f2830aa250c44cc548f747a99259f74eaa Mon Sep 17 00:00:00 2001 From: Philipp Huth Date: Mon, 19 Apr 2021 11:33:55 +0200 Subject: [PATCH 3/6] Add `.pcap.gz` to player file list --- can/io/player.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/can/io/player.py b/can/io/player.py index 99b2f2da1..e8887131c 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -33,6 +33,7 @@ class LogReader(BaseIOHandler): * .db * .log * .pcap + * .pcap.gz Exposes a simple iterator interface, to use simply: @@ -56,6 +57,7 @@ class LogReader(BaseIOHandler): ".db": SqliteReader, ".log": CanutilsLogReader, ".pcap": PcapLogReader, + ".pcap.gz": PcapLogReader, } @staticmethod From 4414bf4508028cca40032fd072012bca8fc08fa3 Mon Sep 17 00:00:00 2001 From: Philipp Huth Date: Tue, 20 Apr 2021 19:34:41 +0200 Subject: [PATCH 4/6] Enable multi suffix usage in player To use with `.pcap.gz` log files or similar --- can/io/player.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/can/io/player.py b/can/io/player.py index e8887131c..d65373a79 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -75,7 +75,8 @@ def __new__(cls, filename: "can.typechecking.StringPathLike", *args, **kwargs): ) LogReader.fetched_plugins = True - suffix = pathlib.PurePath(filename).suffix.lower() + suffix = ''.join(pathlib.PurePath(filename).suffixes) + suffix = suffix.lower() try: return LogReader.message_readers[suffix](filename, *args, **kwargs) except KeyError: From cbf871d32972e212f61c8c2ffc68340c61f86b3f Mon Sep 17 00:00:00 2001 From: Philipp Huth Date: Tue, 20 Apr 2021 21:57:06 +0200 Subject: [PATCH 5/6] Indicate where pcap parser error is raised --- can/io/pcap.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/can/io/pcap.py b/can/io/pcap.py index e5acb0353..fb001960f 100644 --- a/can/io/pcap.py +++ b/can/io/pcap.py @@ -31,6 +31,10 @@ class PcapParseError(Exception): """Pcap file could not be parsed correctly.""" +class PcapHeaderParseError(PcapParseError): + """Pcap header of file could not be parsed correctly.""" +class PcapPacketParseError(PcapParseError): + """Pcap packet within file could not be parsed correctly.""" LOG = logging.getLogger(__name__) @@ -187,13 +191,13 @@ def __init__(self, file): self.endian = "<" self.timestamp_precision = Precision.ns else: - raise PcapParseError("Unexpected endian / precision definition") + raise PcapHeaderParseError("Unexpected endian / precision definition") self.global_header = PcapGlobalHeader(global_header_data, self.endian) # We can only use socketcan packages with generic socketcan and linux-cooked headers if self.global_header.network != DLT_CAN_SOCKETCAN and self.global_header.network != DLT_LINUX_SLL and self.global_header.network != DLT_LINUX_SLL2: - raise PcapParseError("Unsupported link type") + raise PcapHeaderParseError("Unsupported link type") def __iter__(self): @@ -219,7 +223,7 @@ def __iter__(self): sll_header_data ) if protocol_type_field != LINUX_SLL_P_CAN and protocol_type_field != LINUX_SLL_P_CANFD: - raise PcapParseError("Unsupported SLL protocol type") + raise PcapPacketParseError("Unsupported SLL protocol type") elif self.global_header.network == DLT_LINUX_SLL2: sll2_header_data = self.file.read(SLL2_HEADER_STRUCT.size) protocol_type_field, _, _, _, _, _, _, _, _ = SLL2_HEADER_STRUCT.unpack( @@ -227,7 +231,7 @@ def __iter__(self): ) linux_sll_size = SLL2_HEADER_STRUCT.size if protocol_type_field != LINUX_SLL_P_CAN and protocol_type_field != LINUX_SLL_P_CANFD: - raise PcapParseError("Unsupported SLLv2 protocol type") + raise PcapPacketParseError("Unsupported SLLv2 protocol type") # NOTE: Read the header of the CAN frame (8 byte) packet_data_frame_header = self.file.read(SOCKETCAN_HEADER_STRUCT.size) @@ -251,7 +255,7 @@ def __iter__(self): packet_data_frame_length = packet_header.incl_len - (SOCKETCAN_HEADER_STRUCT.size + linux_sll_size) # can_dlc - 8 bit CAN header should be remaining if packet_data_frame_length == (can_dlc - SOCKETCAN_HEADER_STRUCT.size): - raise PcapParseError("Data frame size not equal to CAN frame DLC") + raise PcapPacketParseError("Data frame size not equal to CAN frame DLC") packet_data_frame = self.file.read(packet_data_frame_length) From 354002c9391de0c6df85f64d78c761fa600943d6 Mon Sep 17 00:00:00 2001 From: Philipp Huth Date: Thu, 13 May 2021 23:41:24 +0200 Subject: [PATCH 6/6] Fix issue with filenames including points When we enable files with multiple extension (e.g. `.pcap.gz`), files with points will break the format detection. Therefore, we first test all single extensions and afterwards try to test against 2 extensions. --- can/io/player.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/can/io/player.py b/can/io/player.py index d65373a79..20b45a655 100644 --- a/can/io/player.py +++ b/can/io/player.py @@ -75,14 +75,20 @@ def __new__(cls, filename: "can.typechecking.StringPathLike", *args, **kwargs): ) LogReader.fetched_plugins = True - suffix = ''.join(pathlib.PurePath(filename).suffixes) - suffix = suffix.lower() + suffix = pathlib.PurePath(filename).suffix.lower() try: return LogReader.message_readers[suffix](filename, *args, **kwargs) except KeyError: - raise ValueError( - f'No read support for this unknown log format "{suffix}"' - ) from None + suffix_list = pathlib.PurePath(filename).suffixes + suffix_list_length = len(suffix_list) + try: + suffix = ''.join(suffix_list[suffix_list_length-2:suffix_list_length]) + suffix = suffix.lower() + return LogReader.message_readers[suffix](filename, *args, **kwargs) + except KeyError: + raise ValueError( + f'No read support for this unknown log format "{suffix}"' + ) from None class MessageSync: # pylint: disable=too-few-public-methods