From 748329282a211403f5bcf7e3933c1c4301e67009 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 13 Dec 2017 17:27:19 +0100 Subject: [PATCH 01/60] small changes for better readability --- can/io/sqlite.py | 50 +++++++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 51faf57c3..95d79e4a8 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -1,3 +1,7 @@ +""" +Implements an SQL database writer and reader for storing CAN messages. +""" + from can.listener import BufferedReader from can.message import Message @@ -15,7 +19,7 @@ class SqlReader: def __init__(self, filename): - log.debug("Starting sqlreader with {}".format(filename)) + log.debug("Starting SqlReader with {}".format(filename)) conn = sqlite3.connect(filename) self.c = conn.cursor() @@ -74,18 +78,18 @@ def __init__(self, filename): super(SqliteWriter, self).__init__() self.db_fn = filename self.stop_running_event = threading.Event() - self.writer_thread = threading.Thread(target=self.db_writer_thread) + self.writer_thread = threading.Thread(target=self._db_writer_thread) self.writer_thread.start() def _create_db(self): - # Note you can't share sqlite3 connections between threads + # Note: you can't share sqlite3 connections between threads # hence we setup the db here. - log.info("Creating sqlite db") + log.info("Creating sqlite database") self.conn = sqlite3.connect(self.db_fn) - c = self.conn.cursor() + cursor = self.conn.cursor() # create table structure - c.execute(''' + cursor.execute(''' CREATE TABLE IF NOT EXISTS messages ( ts REAL, @@ -101,7 +105,7 @@ def _create_db(self): self.db_setup = True - def db_writer_thread(self): + def _db_writer_thread(self): num_frames = 0 last_write = time.time() self._create_db() @@ -109,33 +113,36 @@ def db_writer_thread(self): while not self.stop_running_event.is_set(): messages = [] - m = self.get_message(self.GET_MESSAGE_TIMEOUT) - while m is not None: - log.debug("sqlitewriter buffering message") + msg = self.get_message(self.GET_MESSAGE_TIMEOUT) + while msg is not None: + log.debug("SqliteWriter: buffering message") messages.append(( - m.timestamp, - m.arbitration_id, - m.id_type, - m.is_remote_frame, - m.is_error_frame, - m.dlc, - buffer(m.data) + msg.timestamp, + msg.arbitration_id, + msg.id_type, + msg.is_remote_frame, + msg.is_error_frame, + msg.dlc, + buffer(msg.data) )) if time.time() - last_write > self.MAX_TIME_BETWEEN_WRITES: log.debug("Max timeout between writes reached") break - m = self.get_message(self.GET_MESSAGE_TIMEOUT) + msg = self.get_message(self.GET_MESSAGE_TIMEOUT) - if len(messages) > 0: + count = len(messages) + if count > 0: with self.conn: - log.debug("Writing %s frames to db", len(messages)) + log.debug("Writing %s frames to db", count) self.conn.executemany(SqliteWriter.insert_msg_template, messages) - num_frames += len(messages) + num_frames += count last_write = time.time() + # go back up and check if we are still supposed to run + self.conn.close() log.info("Stopped sqlite writer after writing %s messages", num_frames) @@ -143,4 +150,3 @@ def stop(self): self.stop_running_event.set() log.debug("Stopping sqlite writer") self.writer_thread.join() - From 46a7485ee333c34d34b2b4ec49c4bcd0c464cc2a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Wed, 13 Dec 2017 17:46:44 +0100 Subject: [PATCH 02/60] small changes: remove unused parameters & better variable names --- can/io/sqlite.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 95d79e4a8..96368cd79 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -19,23 +19,23 @@ class SqlReader: def __init__(self, filename): - log.debug("Starting SqlReader with {}".format(filename)) + log.debug("Starting SqlReader with %s", filename) conn = sqlite3.connect(filename) - self.c = conn.cursor() + self.cursor = conn.cursor() @staticmethod - def create_frame_from_db_tuple(frame_data): - ts, id, is_extended, is_remote, is_error, dlc, data = frame_data + def _create_frame_from_db_tuple(frame_data): + timestamp, id, is_extended, is_remote, is_error, dlc, data = frame_data return Message( - ts, is_remote, is_extended, is_error, id, dlc, data + timestamp, is_remote, is_extended, is_error, id, dlc, data ) def __iter__(self): log.debug("Iterating through messages from sql db") - for frame_data in self.c.execute("SELECT * FROM messages"): - yield SqlReader.create_frame_from_db_tuple(frame_data) + for frame_data in self.cursor.execute("SELECT * FROM messages"): + yield SqlReader._create_frame_from_db_tuple(frame_data) class SqliteWriter(BufferedReader): @@ -103,8 +103,6 @@ def _create_db(self): ''') self.conn.commit() - self.db_setup = True - def _db_writer_thread(self): num_frames = 0 last_write = time.time() From 095c2a98666cfa4967c24b6af32dcc5f238314e9 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 10:55:44 +0100 Subject: [PATCH 03/60] better readability, fixed sphinx-build warning --- can/interfaces/slcan.py | 95 +++++++++++++++++++++-------------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/can/interfaces/slcan.py b/can/interfaces/slcan.py index 0b5c00fcb..487082501 100755 --- a/can/interfaces/slcan.py +++ b/can/interfaces/slcan.py @@ -1,84 +1,88 @@ """ Interface for slcan compatible interfaces (win32/linux). -(Linux could use slcand/socketcan also). +(Linux could use slcand/socketcan as well). """ + from __future__ import absolute_import -import serial import io import time import logging -from can import CanError, BusABC, Message +import serial +from can import BusABC, Message logger = logging.getLogger(__name__) class slcanBus(BusABC): - """slcan interface""" - - def write(self, str): - if not str.endswith("\r"): - str += "\r" - self.serialPort.write(str.decode()) + """ + slcan interface + """ + + # the supported bitrates and their commands + _BITRATES = { + 10000: 'S0', + 20000: 'S1', + 50000: 'S2', + 100000: 'S3', + 125000: 'S4', + 250000: 'S5', + 500000: 'S6', + 750000: 'S7', + 1000000: 'S8', + 83300: 'S9' + } + + _SLEEP_AFTER_SERIAL_OPEN = 2 # in seconds + + def write(self, string): + if not string.endswith('\r'): + string += '\r' + self.serialPort.write(string.decode()) self.serialPort.flush() def open(self): - self.write("O") + self.write('O') def close(self): - self.write("C") - + self.write('C') - def __init__(self, channel, ttyBaudrate=115200, timeout=1, bitrate=None , **kwargs): + def __init__(self, channel, ttyBaudrate=115200, timeout=1, bitrate=None, **kwargs): """ :param string channel: port of underlying serial or usb device (e.g. /dev/ttyUSB0, COM8, ...) + Must not be empty. :param int ttyBaudrate: baudrate of underlying serial or usb device :param int bitrate: Bitrate in bits/s :param float poll_interval: Poll interval in seconds when reading messages - :param float timeout + :param float timeout: timeout in seconds when reading message """ - - if channel == '': + if not channel: # if not None and not empty raise TypeError("Must specify a serial port.") + if '@' in channel: (channel, ttyBaudrate) = channel.split('@') self.serialPortOrig = serial.Serial(channel, baudrate=ttyBaudrate, timeout=timeout) self.serialPort = io.TextIOWrapper(io.BufferedRWPair(self.serialPortOrig, self.serialPortOrig, 1), - newline='\r', line_buffering=True) + newline='\r', line_buffering=True) + + # why do we sleep here? + time.sleep(self._SLEEP_AFTER_SERIAL_OPEN) - time.sleep(2) if bitrate is not None: self.close() - if bitrate == 10000: - self.write('S0') - elif bitrate == 20000: - self.write('S1') - elif bitrate == 50000: - self.write('S2') - elif bitrate == 100000: - self.write('S3') - elif bitrate == 125000: - self.write('S4') - elif bitrate == 250000: - self.write('S5') - elif bitrate == 500000: - self.write('S6') - elif bitrate == 750000: - self.write('S7') - elif bitrate == 1000000: - self.write('S8') - elif bitrate == 83300: - self.write('S9') + if bitrate in self._BITRATES: + self.write(self._BITRATES[bitrate]) else: - raise ValueError("Invalid bitrate, choose one of 10000 20000 50000 100000 125000 250000 500000 750000 1000000 83300") + # this only prints the keys of the dict + raise ValueError("Invalid bitrate, choose one of " + (', '.join(self._BITRATES)) + '.') self.open() super(slcanBus, self).__init__(channel, **kwargs) @@ -91,25 +95,25 @@ def recv(self, timeout=None): remote = False frame = [] readStr = self.serialPort.readline() - if readStr is None or len(readStr) == 0: + if not readStr: # if not None and not empty return None else: - if readStr[0] == 'T': # entended frame + if readStr[0] == 'T': # extended frame canId = int(readStr[1:9], 16) dlc = int(readStr[9]) extended = True for i in range(0, dlc): frame.append(int(readStr[10 + i * 2:12 + i * 2], 16)) - elif readStr[0] == 't': # normal frame + elif readStr[0] == 't': # normal frame canId = int(readStr[1:4], 16) dlc = int(readStr[4]) for i in range(0, dlc): frame.append(int(readStr[5 + i * 2:7 + i * 2], 16)) extended = False - elif readStr[0] == 'r': # remote frame + elif readStr[0] == 'r': # remote frame canId = int(readStr[1:4], 16) remote = True - elif readStr[0] == 'R': # remote extended frame + elif readStr[0] == 'R': # remote extended frame canId = int(readStr[1:9], 16) extended = True remote = True @@ -140,6 +144,5 @@ def send(self, msg, timeout=None): sendStr += "%02X" % msg.data[i] self.write(sendStr) - def shutdown(self): - self.close() \ No newline at end of file + self.close() From 9b82f358a7bc3f03416ffd48c2fa6fff93cee2a3 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 11:18:16 +0100 Subject: [PATCH 04/60] fixes some docs build issues with sphinx --- doc/conf.py | 7 +++---- doc/listeners.rst | 9 +++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/conf.py b/doc/conf.py index 5d9bc7f05..f56298c53 100755 --- a/doc/conf.py +++ b/doc/conf.py @@ -64,10 +64,9 @@ # The master toctree document. master_doc = 'index' - # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +language = 'en' # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: @@ -131,14 +130,14 @@ #html_logo = None # The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. #html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +#html_static_path = ['_static'] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. diff --git a/doc/listeners.rst b/doc/listeners.rst index 9d2370b06..a7e4ce601 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -59,7 +59,7 @@ SqliteWriter ASC (.asc Logging format) ---------- +------------------------- ASCWriter logs CAN data to an ASCII log file compatible with other CAN tools such as Vector CANalyzer/CANoe and other. Since no official specification exists for the format, it has been reverse- @@ -77,20 +77,21 @@ as further references can-utils can be used: .. autoclass:: can.ASCReader :members: + Log (.log can-utils Logging format) ---------- +----------------------------------- canutilsLogWriter logs CAN data to an ASCII log file compatible with `can-utils ` As specification following references can-utils can be used: `asc2log `_, `log2asc `_. -.. autoclass:: can.canutilsLogWriter +.. autoclass:: can.io.CanutilsLogWriter :members: canutilsLogReader reads CAN data from ASCII log files .log -.. autoclass:: can.canutilsLogReader +.. autoclass:: can.io.CanutilsLogReader :members: From 3f253484d3c4704b88c4bbafa9ac50ae0cf9092f Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 11:19:19 +0100 Subject: [PATCH 05/60] added some info text about "how to build this project", to make getting started faster --- doc/development.rst | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/doc/development.rst b/doc/development.rst index fe6eca046..577fb0657 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -9,6 +9,17 @@ Contribute to source code, documentation, examples and report issues: https://github.com/hardbyte/python-can +Building & Installing +--------------------- + +This assumes that the commands are executed from the root of the repository. + +The project can be built and installed with ``python setup.py build`` and +``python setup.py install``. +The unit tests can be run with ``python setup.py test``. +The docs can be built with ``sphinx-build doc/ doc/_build``. + + Creating a Release ------------------ @@ -48,4 +59,3 @@ The modules in ``python-can`` are: |:doc:`broadcastmanager ` | Contains interface independent broadcast manager | | | code. | +---------------------------------+------------------------------------------------------+ - From 5c9f9cf7b12fafaa32652e57c7d8b32b79d5d5aa Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 15:26:57 +0100 Subject: [PATCH 06/60] allow messages to be used as keys in deicts --- can/message.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/can/message.py b/can/message.py index 8a6765530..c566317e5 100644 --- a/can/message.py +++ b/can/message.py @@ -62,7 +62,7 @@ def __str__(self): if self.data is not None: for index in range(0, min(self.dlc, len(self.data))): data_strings.append("{0:02x}".format(self.data[index])) - if len(data_strings) > 0: + if data_strings: # if not empty field_strings.append(" ".join(data_strings).ljust(24, " ")) else: field_strings.append(" " * 24) @@ -70,7 +70,7 @@ def __str__(self): if (self.data is not None) and (self.data.isalnum()): try: field_strings.append("'{}'".format(self.data.decode('utf-8'))) - except UnicodeError as e: + except UnicodeError: pass return " ".join(field_strings).strip() @@ -100,9 +100,23 @@ def __repr__(self): def __eq__(self, other): return (isinstance(other, self.__class__) and self.arbitration_id == other.arbitration_id and - #self.timestamp == other.timestamp and + #self.timestamp == other.timestamp and # TODO: explain this self.id_type == other.id_type and self.dlc == other.dlc and self.data == other.data and self.is_remote_frame == other.is_remote_frame and self.is_error_frame == other.is_error_frame) + + def __hash__(self): + return hash(( + self.arbitration_id, + # self.timestamp # excluded, like in self.__eq__(self, other) + self.id_type, + self.dlc, + self.data, + self.is_remote_frame, + self.is_error_frame + )) + + def __format__(self, format_spec): + return self.__str__() From bd7e1c2a02980b42ca230c5e66b867db2cf9b40e Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 16:00:53 +0100 Subject: [PATCH 07/60] added some functionality to SqlReader & make sure all writes to the database in SqliteWriter are regulary committed --- can/io/sqlite.py | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 96368cd79..cee64915b 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -1,15 +1,17 @@ """ Implements an SQL database writer and reader for storing CAN messages. -""" -from can.listener import BufferedReader -from can.message import Message +The database schema is given in the documentation of the loggers. +""" import sys import time import threading -import sqlite3 import logging +import sqlite3 + +from can.listener import BufferedReader +from can.message import Message log = logging.getLogger('can.io.sql') @@ -18,25 +20,41 @@ class SqlReader: - def __init__(self, filename): - log.debug("Starting SqlReader with %s", filename) - conn = sqlite3.connect(filename) + """ + Reads recorded CAN messages from a simple SQL database. - self.cursor = conn.cursor() + This class can be iterated over or used to fetch all messages in the + database with :meth:`~SqlReader.read_all`. + """ + _SELECT_ALL_COMMAND = "SELECT * FROM messages" + + def __init__(self, filename): + log.debug("Starting SqlReader with %s", filename) + self.conn = sqlite3.connect(filename) + self.cursor = self.conn.cursor() @staticmethod def _create_frame_from_db_tuple(frame_data): - timestamp, id, is_extended, is_remote, is_error, dlc, data = frame_data + timestamp, can_id, is_extended, is_remote, is_error, dlc, data = frame_data return Message( - timestamp, is_remote, is_extended, is_error, id, dlc, data + timestamp, is_remote, is_extended, is_error, can_id, dlc, data ) def __iter__(self): log.debug("Iterating through messages from sql db") - for frame_data in self.cursor.execute("SELECT * FROM messages"): + for frame_data in self.cursor.execute(self._SELECT_ALL_COMMAND): yield SqlReader._create_frame_from_db_tuple(frame_data) + def read_all(self): + """Fetches all messages in the database.""" + result = self.cursor.execute(self._SELECT_ALL_COMMAND) + return result.fetchall() + + def close(self): + """Closes the connection to the database.""" + self.conn.close() + class SqliteWriter(BufferedReader): """Logs received CAN data to a simple SQL database. @@ -63,10 +81,10 @@ class SqliteWriter(BufferedReader): """ - insert_msg_template = ''' + _INSERT_MSG_TEMPLATE = ''' INSERT INTO messages VALUES (?, ?, ?, ?, ?, ?, ?) - ''' + ''' GET_MESSAGE_TIMEOUT = 0.25 """Number of seconds to wait for messages from internal queue""" @@ -135,7 +153,8 @@ def _db_writer_thread(self): if count > 0: with self.conn: log.debug("Writing %s frames to db", count) - self.conn.executemany(SqliteWriter.insert_msg_template, messages) + self.conn.executemany(SqliteWriter._INSERT_MSG_TEMPLATE, messages) + self.conn.commit() # make the changes visible to the entire database num_frames += count last_write = time.time() From 5b097fcf2e27e899acfc518c53b0d630e245a12d Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 16:01:32 +0100 Subject: [PATCH 08/60] added documentation of the sql database table format to the docs --- doc/listeners.rst | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/listeners.rst b/doc/listeners.rst index a7e4ce601..397c71c5e 100644 --- a/doc/listeners.rst +++ b/doc/listeners.rst @@ -57,6 +57,26 @@ SqliteWriter .. autoclass:: can.SqliteWriter :members: +Database table format +~~~~~~~~~~~~~~~~~~~~~ + +The messages are written to the table ``messages`` in the sqlite database. +The table is created if it does not already exist. + +The entries are as follows: + +============== ============== ============== +Name Data type Note +-------------- -------------- -------------- +ts REAL The timestamp of the message +arbitration_id INTEGER The arbitration id, might use the extended format +extended INTEGER ``1`` if the arbitration id uses the extended format, else ``0`` +remote INTEGER ``1`` if the message is a remote frame, else ``0`` +error INTEGER ``1`` if the message is an error frame, else ``0`` +dlc INTEGER The data length code (DLC) +data BLOB The content of the message +============== ============== ============== + ASC (.asc Logging format) ------------------------- From 8aeaa2bc129c685549b020f2780335d2d810d0e9 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 16:06:31 +0100 Subject: [PATCH 09/60] small changes to the docs (fixes a sphinx-build warning) & added myself to the CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + doc/interfaces/socketcan_native.rst | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 8408ccdc8..851dfbcad 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -19,3 +19,4 @@ Giuseppe Corbelli Christian Sandberg Eduard Bröcker Boris Wenzlaff +Felix Divo diff --git a/doc/interfaces/socketcan_native.rst b/doc/interfaces/socketcan_native.rst index 82738eb25..fd9b8711c 100644 --- a/doc/interfaces/socketcan_native.rst +++ b/doc/interfaces/socketcan_native.rst @@ -5,16 +5,16 @@ Python 3.3 added support for socketcan for linux systems. The socketcan_native interface directly uses Python's socket module to access SocketCAN on linux. This is the most direct route to the kernel -and should provide the most responsive. +and should provide the most responsive one. -The implementation features efficient filtering of can_id's, this filtering +The implementation features efficient filtering of can_id's. That filtering occurs in the kernel and is much much more efficient than filtering messages in Python. Python 3.4 added support for the Broadcast Connection Manager (BCM) -protocol, which if enabled should be used for queueing periodic tasks. +protocol, which - if enabled - should be used for queueing periodic tasks. -Documentation for the socket can backend file can be found: +Documentation for the socketcan back end file can be found: https://www.kernel.org/doc/Documentation/networking/can.txt @@ -41,6 +41,6 @@ bindSocket captureMessage -~~~~~~~~~~~~~ +~~~~~~~~~~~~~~ .. autofunction:: can.interfaces.socketcan.socketcan_native.captureMessage From 3c897060d4e59ee81d2904044dd64459d6ed5a77 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 17:01:32 +0100 Subject: [PATCH 10/60] added length attribute to sqlite table --- can/io/sqlite.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index cee64915b..6267ede19 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -25,6 +25,8 @@ class SqlReader: This class can be iterated over or used to fetch all messages in the database with :meth:`~SqlReader.read_all`. + + Calling len() on this object might not run in constant time. """ _SELECT_ALL_COMMAND = "SELECT * FROM messages" @@ -46,6 +48,10 @@ def __iter__(self): for frame_data in self.cursor.execute(self._SELECT_ALL_COMMAND): yield SqlReader._create_frame_from_db_tuple(frame_data) + def __len__(self): + result = self.cursor.execute("SELECT COUNT(*) FROM messages") + return abs(int(result.fetchone()[0])) + def read_all(self): """Fetches all messages in the database.""" result = self.cursor.execute(self._SELECT_ALL_COMMAND) From 4d0cc6e26485e064aca8be969270a756adf82dcc Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 17:22:20 +0100 Subject: [PATCH 11/60] added more Sqlite tests and refactored some of the others --- can/io/sqlite.py | 1 + test/listener_test.py | 25 ---------------- test/logformats_test.py | 64 +++++++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 53 deletions(-) diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 6267ede19..444d2193a 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -49,6 +49,7 @@ def __iter__(self): yield SqlReader._create_frame_from_db_tuple(frame_data) def __len__(self): + # this might not run in constant time result = self.cursor.execute("SELECT COUNT(*) FROM messages") return abs(int(result.fetchone()[0])) diff --git a/test/listener_test.py b/test/listener_test.py index 2beb66f8c..2c3c71661 100755 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -161,31 +161,6 @@ def testAscListener(self): output_contents = f.read() self.assertTrue('This is some comment' in output_contents) - print("Output from ASCWriter:") - print(output_contents) - - -class FileReaderTest(BusTest): - - def test_sql_reader(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - a_listener = can.SqliteWriter(f.name) - a_listener(generate_message(0xDADADA)) - - sleep(a_listener.MAX_TIME_BETWEEN_WRITES) - while not a_listener.buffer.empty(): - sleep(0.1) - a_listener.stop() - - reader = can.SqlReader(f.name) - - ms = [] - for m in reader: - ms.append(m) - - self.assertEqual(len(ms), 1) - self.assertEqual(0xDADADA, ms[0].arbitration_id) class BLFTest(unittest.TestCase): diff --git a/test/logformats_test.py b/test/logformats_test.py index 043eb97d5..bad158d48 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -1,5 +1,7 @@ import unittest import tempfile +from time import sleep + import can # List of messages of different types that can be used in tests @@ -18,43 +20,49 @@ can.Message(is_error_frame=True, timestamp=1483389466.170), ] +def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=0): + """Tests a pair of writer and reader. -class TestCanutilsLog(unittest.TestCase): + The :attr:`sleep_time` specifies the time to sleep after + writing all messages. + """ + + temp = tempfile.NamedTemporaryFile('w', delete=False) + temp.close() + filename = temp.name + writer = writer_constructor(filename) - def test_reader_writer(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - filename = f.name - writer = can.CanutilsLogWriter(filename) + for msg in TEST_MESSAGES: + writer(msg) - for msg in TEST_MESSAGES: - writer(msg) - writer.stop() + sleep(sleep_time) + writer.stop() - messages = list(can.CanutilsLogReader(filename)) - self.assertEqual(len(messages), len(TEST_MESSAGES)) - for msg1, msg2 in zip(messages, TEST_MESSAGES): - self.assertEqual(msg1, msg2) - self.assertAlmostEqual(msg1.timestamp, msg2.timestamp) + messages = list(reader_constructor(filename)) + test_case.assertEqual(len(messages), len(TEST_MESSAGES)) + for msg1, msg2 in zip(messages, TEST_MESSAGES): + test_case.assertEqual(msg1, msg2) + test_case.assertAlmostEqual(msg1.timestamp, msg2.timestamp) + + +class TestCanutilsLog(unittest.TestCase): + """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" + + def test(self): + _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader) class TestAscFileFormat(unittest.TestCase): + """Tests can.ASCWriter and can.ASCReader""" - def test_reader_writer(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - filename = f.name + def test(self): + _test_writer_and_reader(self, can.ASCWriter, can.ASCReader) - writer = can.ASCWriter(filename) - for msg in TEST_MESSAGES: - writer(msg) - writer.stop() +class TestSqlFileFormat(unittest.TestCase): + """Tests can.SqliteWriter and can.SqliteReader""" + + def test(self): + _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=0.25) - messages = list(can.ASCReader(filename)) - self.assertEqual(len(messages), len(TEST_MESSAGES)) - for msg1, msg2 in zip(messages, TEST_MESSAGES): - self.assertEqual(msg1, msg2) - self.assertAlmostEqual(msg1.timestamp, msg2.timestamp) if __name__ == '__main__': unittest.main() - From 82a2fdf3ec6039ff1d36d138591a13e3d1fa899a Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 17:47:26 +0100 Subject: [PATCH 12/60] added a few test, but some of them fail. Does anyone know why? --- test/logformats_test.py | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index bad158d48..89d3d085c 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -4,20 +4,48 @@ import can +TIME = 1483389946.197 # some random number + # List of messages of different types that can be used in tests TEST_MESSAGES = [ + can.Message(), + can.Message( + data=[1, 2] + ), + can.Message( + arbitration_id=0xAB, extended_id=False + ), + can.Message( + arbitration_id=0x42, extended_id=True + ), can.Message( arbitration_id=0xDADADA, extended_id=True, is_remote_frame=False, - timestamp=1483389464.165, + timestamp=TIME + .165, data=[1, 2, 3, 4, 5, 6, 7, 8]), can.Message( arbitration_id=0x123, extended_id=False, is_remote_frame=False, - timestamp=1483389464.365, + timestamp=TIME + .365, data=[254, 255]), can.Message( arbitration_id=0x768, extended_id=False, is_remote_frame=True, - timestamp=1483389466.165), - can.Message(is_error_frame=True, timestamp=1483389466.170), + timestamp=TIME + 3.165), + can.Message( + is_error_frame=True, + timestamp=TIME + 0.170), + can.Message( + arbitration_id=0xabcdef, extended_id=True, + timestamp=TIME, + data=[1, 2, 3, 4, 5, 6, 7, 8]), + can.Message( + arbitration_id=0x123, extended_id=False, + timestamp=TIME + 42.42, + data=[0xff, 0xff]), + can.Message( + arbitration_id=0xabcdef, extended_id=True, is_remote_frame=True, + timestamp=TIME + 7858.67), + can.Message( + arbitration_id=0xabcdef, is_error_frame=True, + timestamp=TIME + 1.6) ] def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=0): @@ -61,7 +89,7 @@ class TestSqlFileFormat(unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" def test(self): - _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=0.25) + _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=0.5) if __name__ == '__main__': From 65e18007a3284851e1a4ce1d1d51e116fc162d45 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 22:25:05 +0100 Subject: [PATCH 13/60] fixed a crash of ASCReader if the data field was empty --- can/interfaces/slcan.py | 2 +- can/io/asc.py | 91 +++++++++++++++++++++++++---------------- test/logformats_test.py | 55 +++++++++++++++++-------- 3 files changed, 96 insertions(+), 52 deletions(-) diff --git a/can/interfaces/slcan.py b/can/interfaces/slcan.py index 487082501..ad2f24197 100755 --- a/can/interfaces/slcan.py +++ b/can/interfaces/slcan.py @@ -63,7 +63,7 @@ def __init__(self, channel, ttyBaudrate=115200, timeout=1, bitrate=None, **kwarg timeout in seconds when reading message """ - if not channel: # if not None and not empty + if not channel: # if None or empty raise TypeError("Must specify a serial port.") if '@' in channel: diff --git a/can/io/asc.py b/can/io/asc.py index 6b800d44b..e40209570 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -1,61 +1,82 @@ +from datetime import datetime +import time +import logging + from can.listener import Listener from can.message import Message -from datetime import datetime -import time CAN_MSG_EXT = 0x80000000 CAN_ID_MASK = 0x1FFFFFFF +logger = logging.getLogger(__name__) + class ASCReader(object): """ Iterator of CAN messages from a ASC Logging File. """ def __init__(self, filename): - self.fp = open(filename, "r") + self.file = open(filename, 'r') + + @staticmethod + def _extract_can_id(str_can_id): + if str_can_id[-1:].lower() == "x": + is_extended = True + can_id = int(str_can_id[0:-1], 16) + else: + is_extended = False + can_id = int(str_can_id, 16) + #logging.debug('ASCReader: _extract_can_id("%s") -> %x, %r', str_can_id, can_id, is_extended) + return (can_id, is_extended) def __iter__(self): - def extractCanId(strCanId): - if strCanId[-1:].lower() == "x": - isExtended = True - can_id = int(strCanId[0:-1], 16) - else: - isExtended = False - can_id = int(strCanId, 16) - return (can_id, isExtended) + for line in self.file: + #logger.debug("ASCReader: parsing line: '%s'", line.splitlines()[0]) - for line in self.fp: temp = line.strip() - if len(temp) == 0 or not temp[0].isdigit(): + if not temp or not temp[0].isdigit(): continue - (time, channel, dummy) = temp.split(None,2) # , frameType, dlc, frameData + (timestamp, channel, dummy) = temp.split(None, 2) # , frameType, dlc, frameData + timestamp = float(timestamp) - time = float(time) - if dummy.strip()[0:10] == "ErrorFrame": - time = float(time) - msg = Message(timestamp=time, is_error_frame=True) + if dummy.strip()[0:10] == 'ErrorFrame': + msg = Message(timestamp=timestamp, is_error_frame=True) yield msg - continue - if not channel.isdigit() or dummy.strip()[0:10] == "Statistic:": - continue - if dummy[-1:].lower() == "r": - (canId, _) = dummy.split(None, 1) - msg = Message(timestamp=time, - arbitration_id=extractCanId(canId)[0] & CAN_ID_MASK, - extended_id=extractCanId(canId)[1], + + elif not channel.isdigit() or dummy.strip()[0:10] == 'Statistic:': + pass + + elif dummy[-1:].lower() == 'r': + (can_id_str, _) = dummy.split(None, 1) + (can_id_num, is_extended_id) = self._extract_can_id(can_id_str) + msg = Message(timestamp=timestamp, + arbitration_id=can_id_num & CAN_ID_MASK, + extended_id=is_extended_id, is_remote_frame=True) yield msg + else: - (canId, direction,_,dlc,data) = dummy.split(None,4) + try: + # this only works if dlc > 0 and thus data is availabe + (can_id_str, _, _, dlc, data) = dummy.split(None, 4) + except ValueError: + # but if not, we only want to get the stuff up to the dlc + (can_id_str, _, _, dlc ) = dummy.split(None, 3) + # and we set data to an empty sequence manually + data = '' dlc = int(dlc) frame = bytearray() data = data.split() for byte in data[0:dlc]: - frame.append(int(byte,16)) - msg = Message(timestamp=time, - arbitration_id=extractCanId(canId)[0] & CAN_ID_MASK, - extended_id=extractCanId(canId)[1], + frame.append(int(byte, 16)) + + (can_id_num, is_extended_id) = self._extract_can_id(can_id_str) + + msg = Message( + timestamp=timestamp, + arbitration_id=can_id_num & CAN_ID_MASK, + extended_id=is_extended_id, is_remote_frame=False, dlc=dlc, data=frame) @@ -65,14 +86,14 @@ def extractCanId(strCanId): class ASCWriter(Listener): """Logs CAN data to an ASCII log file (.asc)""" - LOG_STRING = "{time: 9.4f} {channel} {id:<15} Rx {dtype} {data}\n" - EVENT_STRING = "{time: 9.4f} {message}\n" + LOG_STRING = "{time: 9.4f} {channel} {id:<15} Rx {dtype} {data}\n" + EVENT_STRING = "{time: 9.4f} {message}\n" def __init__(self, filename, channel=1): now = datetime.now().strftime("%a %b %m %I:%M:%S %p %Y") self.channel = channel self.started = time.time() - self.log_file = open(filename, "w") + self.log_file = open(filename, 'w') self.log_file.write("date %s\n" % now) self.log_file.write("base hex timestamps absolute\n") self.log_file.write("internal events logged\n") @@ -97,7 +118,7 @@ def log_event(self, message, timestamp=None): def on_message_received(self, msg): if msg.is_error_frame: - self.log_event("{} ErrorFrame".format(self.channel), msg.timestamp) + self.log_event("{} ErrorFrame".format(self.channel), msg.timestamp) return if msg.is_remote_frame: diff --git a/test/logformats_test.py b/test/logformats_test.py index 89d3d085c..9921edb62 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -8,44 +8,61 @@ # List of messages of different types that can be used in tests TEST_MESSAGES = [ - can.Message(), can.Message( - data=[1, 2] + # empty ), can.Message( + # only data + data=[0x00, 0x42] + ), + can.Message( + # no data arbitration_id=0xAB, extended_id=False ), can.Message( + # no data arbitration_id=0x42, extended_id=True ), + can.Message( + # empty data + data=[] + ), can.Message( arbitration_id=0xDADADA, extended_id=True, is_remote_frame=False, timestamp=TIME + .165, - data=[1, 2, 3, 4, 5, 6, 7, 8]), + data=[1, 2, 3, 4, 5, 6, 7, 8] + ), can.Message( arbitration_id=0x123, extended_id=False, is_remote_frame=False, timestamp=TIME + .365, - data=[254, 255]), + data=[254, 255] + ), can.Message( arbitration_id=0x768, extended_id=False, is_remote_frame=True, - timestamp=TIME + 3.165), + timestamp=TIME + 3.165 + ), can.Message( is_error_frame=True, - timestamp=TIME + 0.170), + timestamp=TIME + 0.170 + ), can.Message( - arbitration_id=0xabcdef, extended_id=True, + arbitration_id=0xABCDEF, extended_id=True, timestamp=TIME, - data=[1, 2, 3, 4, 5, 6, 7, 8]), + data=[1, 2, 3, 4, 5, 6, 7, 8] + ), can.Message( arbitration_id=0x123, extended_id=False, timestamp=TIME + 42.42, - data=[0xff, 0xff]), + data=[0xff, 0xff] + ), can.Message( - arbitration_id=0xabcdef, extended_id=True, is_remote_frame=True, - timestamp=TIME + 7858.67), + arbitration_id=0xABCDEF, extended_id=True, is_remote_frame=True, + timestamp=TIME + 7858.67 + ), can.Message( - arbitration_id=0xabcdef, is_error_frame=True, - timestamp=TIME + 1.6) + arbitration_id=0xABCDEF, is_error_frame=True, + timestamp=TIME + 1.6 + ), ] def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=0): @@ -68,9 +85,15 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s messages = list(reader_constructor(filename)) test_case.assertEqual(len(messages), len(TEST_MESSAGES)) - for msg1, msg2 in zip(messages, TEST_MESSAGES): - test_case.assertEqual(msg1, msg2) - test_case.assertAlmostEqual(msg1.timestamp, msg2.timestamp) + + for i, (msg1, msg2) in enumerate(zip(messages, TEST_MESSAGES)): + try: + test_case.assertEqual(msg1, msg2) + test_case.assertAlmostEqual(msg1.timestamp, msg2.timestamp) + except Exception as exception: + # attach the index + exception.args += ("test failed at index #{}".format(i), ) + raise exception class TestCanutilsLog(unittest.TestCase): From 682e055db1b1c386890c2d5f6a627678b66aaec3 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Thu, 14 Dec 2017 23:49:17 +0100 Subject: [PATCH 14/60] fixed crashing tests --- test/logformats_test.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 9921edb62..39d4024be 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -58,11 +58,7 @@ can.Message( arbitration_id=0xABCDEF, extended_id=True, is_remote_frame=True, timestamp=TIME + 7858.67 - ), - can.Message( - arbitration_id=0xABCDEF, is_error_frame=True, - timestamp=TIME + 1.6 - ), + ) ] def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=0): @@ -86,10 +82,10 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s messages = list(reader_constructor(filename)) test_case.assertEqual(len(messages), len(TEST_MESSAGES)) - for i, (msg1, msg2) in enumerate(zip(messages, TEST_MESSAGES)): + for i, (read, original) in enumerate(zip(messages, TEST_MESSAGES)): try: - test_case.assertEqual(msg1, msg2) - test_case.assertAlmostEqual(msg1.timestamp, msg2.timestamp) + test_case.assertEqual(read, original) + test_case.assertAlmostEqual(read.timestamp, original.timestamp) except Exception as exception: # attach the index exception.args += ("test failed at index #{}".format(i), ) @@ -99,19 +95,19 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s class TestCanutilsLog(unittest.TestCase): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" - def test(self): + def test_writer_and_reader(self): _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader) class TestAscFileFormat(unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" - def test(self): + def test_writer_and_reader(self): _test_writer_and_reader(self, can.ASCWriter, can.ASCReader) class TestSqlFileFormat(unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" - def test(self): + def test_writer_and_reader(self): _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=0.5) From 62b188dddbfb89dd879df8be4304c1dee8d31be5 Mon Sep 17 00:00:00 2001 From: pierreluctg Date: Thu, 11 Jan 2018 09:37:06 -0500 Subject: [PATCH 15/60] Fix the KeyError: 'interface' Existing code throw in a `KeyError: 'interface'` when a invalid interface name is used. --- can/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/can/util.py b/can/util.py index 66c468f41..dd1bf67a1 100644 --- a/can/util.py +++ b/can/util.py @@ -156,7 +156,7 @@ def load_config(path=None, config=None): system_config['interface'] = choose_socketcan_implementation() if system_config['interface'] not in VALID_INTERFACES: - raise NotImplementedError('Invalid CAN Bus Type - {}'.format(can.rc['interface'])) + raise NotImplementedError('Invalid CAN Bus Type - {}'.format(system_config['interface'])) if 'bitrate' in system_config: system_config['bitrate'] = int(system_config['bitrate']) From 1d3e3737fd9b226ca447f83b42b3272430e3c595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Tue, 9 Jan 2018 08:49:08 -0500 Subject: [PATCH 16/60] Replace neovi inteface inplementation Replacing the pyneovi implementation with ICS own python interface python-ics. `neovi` interface is replaced by `icsneovi` interface. `neovi` interface is now pointing to the the `icsneovi` interface for compatibility. --- can/interface.py | 3 +- can/interfaces/__init__.py | 3 +- can/interfaces/ics_neovi/__init__.py | 1 + can/interfaces/ics_neovi/neovi_bus.py | 255 ++++++++++++++++++++++++++ can/interfaces/neovi_api/__init__.py | 1 - can/interfaces/neovi_api/neovi_api.py | 122 ------------ doc/interfaces.rst | 2 +- doc/interfaces/ics_neovi.rst | 46 +++++ doc/interfaces/neovi.rst | 54 ------ setup.py | 3 +- 10 files changed, 309 insertions(+), 181 deletions(-) create mode 100644 can/interfaces/ics_neovi/__init__.py create mode 100644 can/interfaces/ics_neovi/neovi_bus.py delete mode 100644 can/interfaces/neovi_api/__init__.py delete mode 100644 can/interfaces/neovi_api/neovi_api.py create mode 100644 doc/interfaces/ics_neovi.rst delete mode 100644 doc/interfaces/neovi.rst diff --git a/can/interface.py b/can/interface.py index 84758a448..9e2e5d2e6 100644 --- a/can/interface.py +++ b/can/interface.py @@ -18,7 +18,8 @@ 'nican': ('can.interfaces.nican', 'NicanBus'), 'iscan': ('can.interfaces.iscan', 'IscanBus'), 'virtual': ('can.interfaces.virtual', 'VirtualBus'), - 'neovi': ('can.interfaces.neovi_api', 'NeoVIBus'), + 'neovi': ('can.interfaces.ics_neovi', 'NeoViBus'), + 'icsneovi': ('can.interfaces.ics_neovi', 'NeoViBus'), 'vector': ('can.interfaces.vector', 'VectorBus'), 'slcan': ('can.interfaces.slcan', 'slcanBus') } diff --git a/can/interfaces/__init__.py b/can/interfaces/__init__.py index 942eb563a..6af4ee778 100644 --- a/can/interfaces/__init__.py +++ b/can/interfaces/__init__.py @@ -5,4 +5,5 @@ VALID_INTERFACES = set(['kvaser', 'serial', 'pcan', 'socketcan_native', 'socketcan_ctypes', 'socketcan', 'usb2can', 'ixxat', - 'nican', 'iscan', 'vector', 'virtual', 'neovi','slcan']) + 'nican', 'iscan', 'vector', 'virtual', 'neovi', + 'icsneovi', 'slcan']) diff --git a/can/interfaces/ics_neovi/__init__.py b/can/interfaces/ics_neovi/__init__.py new file mode 100644 index 000000000..5b1aa2052 --- /dev/null +++ b/can/interfaces/ics_neovi/__init__.py @@ -0,0 +1 @@ +from can.interfaces.ics_neovi.neovi_bus import NeoViBus diff --git a/can/interfaces/ics_neovi/neovi_bus.py b/can/interfaces/ics_neovi/neovi_bus.py new file mode 100644 index 000000000..331ab9221 --- /dev/null +++ b/can/interfaces/ics_neovi/neovi_bus.py @@ -0,0 +1,255 @@ +""" +ICS NeoVi interface module. + +python-ics is a Python wrapper around the API provided by Intrepid Control +Systems for communicating with their NeoVI range of devices. + +Implementation references: +* https://github.com/intrepidcs/python_ics +""" + +import logging + +try: + import queue +except ImportError: + import Queue as queue + +from can import Message +from can.bus import BusABC + +logger = logging.getLogger(__name__) + +try: + import ics +except ImportError: + logger.error( + "You won't be able to use the ICS NeoVi can backend without the " + "python-ics module installed!" + ) + ics = None + + +class NeoViBus(BusABC): + """ + The CAN Bus implemented for the python_ics interface + https://github.com/intrepidcs/python_ics + """ + + def __init__(self, channel=None, can_filters=None, **config): + """ + + :param int channel: + The Channel id to create this bus with. + :param list can_filters: + A list of dictionaries each containing a "can_id" and a "can_mask". + :param use_system_timestamp: + Use system timestamp for can messages instead of the hardware time + stamp + + >>> [{"can_id": 0x11, "can_mask": 0x21}] + + """ + super(NeoViBus, self).__init__(channel, can_filters, **config) + if ics is None: + raise Exception('Please install python-ics') + + logger.info("CAN Filters: {}".format(can_filters)) + logger.info("Got configuration of: {}".format(config)) + + self._use_system_timestamp = bool( + config.get('use_system_timestamp', False) + ) + + try: + channel = int(channel) + except ValueError: + raise ValueError('channel must be an integer') + + type_filter = config.get('type_filter') + serial = config.get('serial') + self.dev = self._open_device(type_filter, serial) + + self.channel_info = '%s %s CH:%s' % ( + self.dev.Name, + self.dev.SerialNumber, + channel + ) + logger.info("Using device: {}".format(self.channel_info)) + + ics.load_default_settings(self.dev) + + self.sw_filters = None + self.set_filters(can_filters) + self.rx_buffer = queue.Queue() + self.opened = True + + self.network = int(channel) if channel is not None else None + + # TODO: Change the scaling based on the device type + self.ts_scaling = ( + ics.NEOVI6_VCAN_TIMESTAMP_1, ics.NEOVI6_VCAN_TIMESTAMP_2 + ) + + def shutdown(self): + super(NeoViBus, self).shutdown() + self.opened = False + ics.close_device(self.dev) + + def _open_device(self, type_filter=None, serial=None): + if type_filter is not None: + devices = ics.find_devices(type_filter) + else: + devices = ics.find_devices() + + for device in devices: + if serial is None: + dev = device + break + if str(device.SerialNumber) == serial: + dev = device + break + else: + msg = ['No device'] + + if type_filter is not None: + msg.append('with type {}'.format(type_filter)) + if serial is not None: + msg.append('with serial {}'.format(serial)) + msg.append('found.') + raise Exception(' '.join(msg)) + ics.open_device(dev) + return dev + + def _process_msg_queue(self, timeout=None): + try: + messages, errors = ics.get_messages(self.dev, False, timeout) + except ics.RuntimeError: + return + for ics_msg in messages: + if ics_msg.NetworkID != self.network: + continue + if ics_msg.ArbIDOrHeader == 0: + # Looks like ICS device sends frames with ArbIDOrHeader = 0 + # Need to find out exactly what they are for + # Filtering them for now + continue + if not self._is_filter_match(ics_msg.ArbIDOrHeader): + continue + self.rx_buffer.put(ics_msg) + if errors: + logger.warning("%d errors found" % errors) + + for msg in ics.get_error_messages(self.dev): + logger.warning(msg) + + def _is_filter_match(self, arb_id): + """ + If SW filtering is used, checks if the `arb_id` matches any of + the filters setup. + + :param int arb_id: + CAN ID to check against. + + :return: + True if `arb_id` matches any filters + (or if SW filtering is not used). + """ + if not self.sw_filters: + # Filtering done on HW or driver level or no filtering + return True + for can_filter in self.sw_filters: + if not (arb_id ^ can_filter['can_id']) & can_filter['can_mask']: + return True + return False + + def _get_timestamp_for_msg(self, ics_msg): + if self._use_system_timestamp: + # This is the system time stamp. + # TimeSystem is loaded with the value received from the timeGetTime + # call in the WIN32 multimedia API. + # + # The timeGetTime accuracy is up to 1 millisecond. See the WIN32 + # API documentation for more information. + # + # This timestamp is useful for time comparing with other system + # events or data which is not synced with the neoVI timestamp. + # + # Currently, TimeSystem2 is not used. + return ics_msg.TimeSystem + else: + # This is the hardware time stamp. + # The TimeStamp is reset to zero every time the OpenPort method is + # called. + return \ + float(ics_msg.TimeHardware2) * self.ts_scaling[1] + \ + float(ics_msg.TimeHardware) * self.ts_scaling[0] + + def _ics_msg_to_message(self, ics_msg): + return Message( + timestamp=self._get_timestamp_for_msg(ics_msg), + arbitration_id=ics_msg.ArbIDOrHeader, + data=ics_msg.Data[:ics_msg.NumberBytesData], + dlc=ics_msg.NumberBytesData, + extended_id=bool( + ics_msg.StatusBitField & ics.SPY_STATUS_XTD_FRAME + ), + is_remote_frame=bool( + ics_msg.StatusBitField & ics.SPY_STATUS_REMOTE_FRAME + ) + ) + + def recv(self, timeout=None): + try: + self._process_msg_queue(timeout=timeout) + ics_msg = self.rx_buffer.get_nowait() + self.rx_buffer.task_done() + return self._ics_msg_to_message(ics_msg) + except queue.Empty: + pass + return None + + def send(self, msg, timeout=None): + if not self.opened: + return + data = tuple(msg.data) + + flags = 0 + if msg.is_extended_id: + flags |= ics.SPY_STATUS_XTD_FRAME + if msg.is_remote_frame: + flags |= ics.SPY_STATUS_REMOTE_FRAME + + message = ics.SpyMessage() + message.ArbIDOrHeader = msg.arbitration_id + message.NumberBytesData = len(data) + message.Data = data + message.StatusBitField = flags + message.StatusBitField2 = 0 + message.NetworkID = self.network + ics.transmit_messages(self.dev, message) + + def set_filters(self, can_filters=None): + """Apply filtering to all messages received by this Bus. + + Calling without passing any filters will reset the applied filters. + + :param list can_filters: + A list of dictionaries each containing a "can_id" and a "can_mask". + + >>> [{"can_id": 0x11, "can_mask": 0x21}] + + A filter matches, when + `` & can_mask == can_id & can_mask`` + + """ + self.sw_filters = can_filters or [] + + if not len(self.sw_filters): + logger.info("Filtering has been disabled") + else: + for can_filter in can_filters: + can_id = can_filter["can_id"] + can_mask = can_filter["can_mask"] + logger.info( + "Filtering on ID 0x%X, mask 0x%X", can_id, can_mask) diff --git a/can/interfaces/neovi_api/__init__.py b/can/interfaces/neovi_api/__init__.py deleted file mode 100644 index 9a9e7ff02..000000000 --- a/can/interfaces/neovi_api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from can.interfaces.neovi_api.neovi_api import NeoVIBus diff --git a/can/interfaces/neovi_api/neovi_api.py b/can/interfaces/neovi_api/neovi_api.py deleted file mode 100644 index 4a8aa2044..000000000 --- a/can/interfaces/neovi_api/neovi_api.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -pyneovi interface module. - -pyneovi is a Python wrapper around the API provided by Intrepid Control Systems -for communicating with their NeoVI range of devices. - -Implementation references: -* http://pyneovi.readthedocs.io/en/latest/ -* https://bitbucket.org/Kemp_J/pyneovi -""" - -import logging - -logger = logging.getLogger(__name__) - -try: - import queue -except ImportError: - import Queue as queue - -try: - from neovi import neodevice - from neovi import neovi - from neovi.structures import icsSpyMessage -except ImportError as e: - logger.warning("Cannot load pyneovi: %s", e) - -from can import Message -from can.bus import BusABC - - -SPY_STATUS_XTD_FRAME = 0x04 -SPY_STATUS_REMOTE_FRAME = 0x08 - - -def neo_device_name(device_type): - names = { - neovi.NEODEVICE_BLUE: 'neoVI BLUE', - neovi.NEODEVICE_DW_VCAN: 'ValueCAN', - neovi.NEODEVICE_FIRE: 'neoVI FIRE', - neovi.NEODEVICE_VCAN3: 'ValueCAN3', - neovi.NEODEVICE_YELLOW: 'neoVI YELLOW', - neovi.NEODEVICE_RED: 'neoVI RED', - neovi.NEODEVICE_ECU: 'neoECU', - # neovi.NEODEVICE_IEVB: '' - } - return names.get(device_type, 'Unknown neoVI') - - -class NeoVIBus(BusABC): - """ - The CAN Bus implemented for the pyneovi interface. - """ - - def __init__(self, channel=None, can_filters=None, **config): - """ - - :param int channel: - The Channel id to create this bus with. - """ - type_filter = config.get('type_filter', neovi.NEODEVICE_ALL) - neodevice.init_api() - self.device = neodevice.find_devices(type_filter)[0] - self.device.open() - self.channel_info = '%s %s on channel %s' % ( - neo_device_name(self.device.get_type()), - self.device.device.SerialNumber, - channel - ) - - self.rx_buffer = queue.Queue() - - self.network = int(channel) if channel is not None else None - self.device.subscribe_to(self._rx_buffer, network=self.network) - - def __del__(self): - self.shutdown() - - def shutdown(self): - self.device.pump_messages = False - if self.device.msg_queue_thread is not None: - self.device.msg_queue_thread.join() - - def _rx_buffer(self, msg, user_data): - self.rx_buffer.put_nowait(msg) - - def _ics_msg_to_message(self, ics_msg): - return Message( - timestamp=neovi.GetTimeStampForMsg(self.device.handle, ics_msg)[1], - arbitration_id=ics_msg.ArbIDOrHeader, - data=ics_msg.Data[:ics_msg.NumberBytesData], - dlc=ics_msg.NumberBytesData, - extended_id=bool(ics_msg.StatusBitField & - SPY_STATUS_XTD_FRAME), - is_remote_frame=bool(ics_msg.StatusBitField & - SPY_STATUS_REMOTE_FRAME), - ) - - def recv(self, timeout=None): - try: - ics_msg = self.rx_buffer.get(block=True, timeout=timeout) - except queue.Empty: - pass - else: - if ics_msg.NetworkID == self.network: - return self._ics_msg_to_message(ics_msg) - - def send(self, msg, timeout=None): - data = tuple(msg.data) - flags = SPY_STATUS_XTD_FRAME if msg.is_extended_id else 0 - if msg.is_remote_frame: - flags |= SPY_STATUS_REMOTE_FRAME - - ics_msg = icsSpyMessage() - ics_msg.ArbIDOrHeader = msg.arbitration_id - ics_msg.NumberBytesData = len(data) - ics_msg.Data = data - ics_msg.StatusBitField = flags - ics_msg.StatusBitField2 = 0 - ics_msg.DescriptionID = self.device.tx_id - self.device.tx_id += 1 - self.device.tx_raw_message(ics_msg, self.network) diff --git a/doc/interfaces.rst b/doc/interfaces.rst index b150f6cfb..c98c5aa02 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -20,7 +20,7 @@ The available interfaces are: interfaces/usb2can interfaces/nican interfaces/iscan - interfaces/neovi + interfaces/ics_neovi interfaces/vector interfaces/virtual diff --git a/doc/interfaces/ics_neovi.rst b/doc/interfaces/ics_neovi.rst new file mode 100644 index 000000000..f1463c205 --- /dev/null +++ b/doc/interfaces/ics_neovi.rst @@ -0,0 +1,46 @@ +ICSNEOVI Interface +================== + +.. warning:: + + This ``ICS NeoVI`` documentation is a work in progress. Feedback and revisions + are most welcome! + + +Interface to `Intrepid Control Systems `__ neoVI +API range of devices via `python-ics `__ +wrapper on Windows. + + +Installation +------------ +This icsneovi interface requires the installation of the ICS neoVI DLL and python-ics +package. + +- Download and install the Intrepid Product Drivers + `Intrepid Product Drivers `__ + +- Install python-ics + .. code-block:: bash + + pip install python-ics + + +Configuration +------------- + +An example `can.ini` file for windows 7: + +:: + + [default] + interface = icsneovi + channel = 1 + + +Bus +--- + +.. autoclass:: can.interfaces.ics_neovi.NeoViBus + + diff --git a/doc/interfaces/neovi.rst b/doc/interfaces/neovi.rst deleted file mode 100644 index 48f4ef1d9..000000000 --- a/doc/interfaces/neovi.rst +++ /dev/null @@ -1,54 +0,0 @@ -neoVI Interface -=============== - -.. warning:: - - This ``neoVI`` documentation is a work in progress. Feedback and revisions - are most welcome! - - -Interface to `Intrepid Control Systems `__ neoVI -API range of devices via `pyneovi `__ -wrapper on Windows. - -.. note:: - - This interface is not supported on Linux, however on Linux neoVI devices - are supported via :doc:`socketcan` with ICS `Kernel-mode SocketCAN module - for Intrepid devices - `__ and - `icsscand `__ - - -Installation ------------- -This neoVI interface requires the installation of the ICS neoVI DLL and pyneovi -package. - -- Download and install the Intrepid Product Drivers - `Intrepid Product Drivers `__ - -- Install pyneovi using pip and the pyneovi bitbucket repo: - .. code-block:: bash - - pip install https://bitbucket.org/Kemp_J/pyneovi/get/default.zip - - -Configuration -------------- - -An example `can.ini` file for windows 7: - -:: - - [default] - interface = neovi - channel = 1 - - -Bus ---- - -.. autoclass:: can.interfaces.neovi_api.NeoVIBus - - diff --git a/setup.py b/setup.py index da2541c50..c905e3a72 100644 --- a/setup.py +++ b/setup.py @@ -30,6 +30,7 @@ test_suite="nose.collector", tests_require=['mock', 'nose', 'pyserial'], extras_require={ - 'serial': ['pyserial'] + 'serial': ['pyserial'], + 'icsneovi': ['python-ics'], } ) From 860a74fe277a4574828c3409a4dc84f55a08faa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Wed, 10 Jan 2018 11:42:27 -0500 Subject: [PATCH 17/60] Renamed ics_neovi interface to neovi --- can/interface.py | 1 - can/interfaces/__init__.py | 2 +- doc/interfaces.rst | 2 +- doc/interfaces/{ics_neovi.rst => neovi.rst} | 6 +++--- setup.py | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) rename doc/interfaces/{ics_neovi.rst => neovi.rst} (85%) diff --git a/can/interface.py b/can/interface.py index 9e2e5d2e6..1ac7954b1 100644 --- a/can/interface.py +++ b/can/interface.py @@ -19,7 +19,6 @@ 'iscan': ('can.interfaces.iscan', 'IscanBus'), 'virtual': ('can.interfaces.virtual', 'VirtualBus'), 'neovi': ('can.interfaces.ics_neovi', 'NeoViBus'), - 'icsneovi': ('can.interfaces.ics_neovi', 'NeoViBus'), 'vector': ('can.interfaces.vector', 'VectorBus'), 'slcan': ('can.interfaces.slcan', 'slcanBus') } diff --git a/can/interfaces/__init__.py b/can/interfaces/__init__.py index 6af4ee778..ae1685c0d 100644 --- a/can/interfaces/__init__.py +++ b/can/interfaces/__init__.py @@ -6,4 +6,4 @@ VALID_INTERFACES = set(['kvaser', 'serial', 'pcan', 'socketcan_native', 'socketcan_ctypes', 'socketcan', 'usb2can', 'ixxat', 'nican', 'iscan', 'vector', 'virtual', 'neovi', - 'icsneovi', 'slcan']) + 'slcan']) diff --git a/doc/interfaces.rst b/doc/interfaces.rst index c98c5aa02..b150f6cfb 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -20,7 +20,7 @@ The available interfaces are: interfaces/usb2can interfaces/nican interfaces/iscan - interfaces/ics_neovi + interfaces/neovi interfaces/vector interfaces/virtual diff --git a/doc/interfaces/ics_neovi.rst b/doc/interfaces/neovi.rst similarity index 85% rename from doc/interfaces/ics_neovi.rst rename to doc/interfaces/neovi.rst index f1463c205..dbb753479 100644 --- a/doc/interfaces/ics_neovi.rst +++ b/doc/interfaces/neovi.rst @@ -1,4 +1,4 @@ -ICSNEOVI Interface +NEOVI Interface ================== .. warning:: @@ -14,7 +14,7 @@ wrapper on Windows. Installation ------------ -This icsneovi interface requires the installation of the ICS neoVI DLL and python-ics +This neovi interface requires the installation of the ICS neoVI DLL and python-ics package. - Download and install the Intrepid Product Drivers @@ -34,7 +34,7 @@ An example `can.ini` file for windows 7: :: [default] - interface = icsneovi + interface = neovi channel = 1 diff --git a/setup.py b/setup.py index c905e3a72..8813466ed 100644 --- a/setup.py +++ b/setup.py @@ -31,6 +31,6 @@ tests_require=['mock', 'nose', 'pyserial'], extras_require={ 'serial': ['pyserial'], - 'icsneovi': ['python-ics'], + 'neovi': ['python-ics'], } ) From 7dcf8d8bed23525b1f432ed4efc5a75e75b8422d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 11 Jan 2018 09:59:23 -0500 Subject: [PATCH 18/60] Change the rx queue to a deque. Change the rx queue to a deque. Removed the filtering of the abr id 0. Adding the channel information to the rx messages. Changed the import error logging from error to warning. --- can/interfaces/ics_neovi/neovi_bus.py | 41 ++++++++++++--------------- 1 file changed, 18 insertions(+), 23 deletions(-) diff --git a/can/interfaces/ics_neovi/neovi_bus.py b/can/interfaces/ics_neovi/neovi_bus.py index 331ab9221..1a4b2a455 100644 --- a/can/interfaces/ics_neovi/neovi_bus.py +++ b/can/interfaces/ics_neovi/neovi_bus.py @@ -9,11 +9,7 @@ """ import logging - -try: - import queue -except ImportError: - import Queue as queue +from collections import deque from can import Message from can.bus import BusABC @@ -22,10 +18,10 @@ try: import ics -except ImportError: - logger.error( +except ImportError as ie: + logger.warning( "You won't be able to use the ICS NeoVi can backend without the " - "python-ics module installed!" + "python-ics module installed!: %s", ie ) ics = None @@ -52,7 +48,7 @@ def __init__(self, channel=None, can_filters=None, **config): """ super(NeoViBus, self).__init__(channel, can_filters, **config) if ics is None: - raise Exception('Please install python-ics') + raise ImportError('Please install python-ics') logger.info("CAN Filters: {}".format(can_filters)) logger.info("Got configuration of: {}".format(config)) @@ -61,6 +57,7 @@ def __init__(self, channel=None, can_filters=None, **config): config.get('use_system_timestamp', False) ) + # TODO: Add support for multiples channels try: channel = int(channel) except ValueError: @@ -81,7 +78,7 @@ def __init__(self, channel=None, can_filters=None, **config): self.sw_filters = None self.set_filters(can_filters) - self.rx_buffer = queue.Queue() + self.rx_buffer = deque() self.opened = True self.network = int(channel) if channel is not None else None @@ -129,14 +126,9 @@ def _process_msg_queue(self, timeout=None): for ics_msg in messages: if ics_msg.NetworkID != self.network: continue - if ics_msg.ArbIDOrHeader == 0: - # Looks like ICS device sends frames with ArbIDOrHeader = 0 - # Need to find out exactly what they are for - # Filtering them for now - continue if not self._is_filter_match(ics_msg.ArbIDOrHeader): continue - self.rx_buffer.put(ics_msg) + self.rx_buffer.append(ics_msg) if errors: logger.warning("%d errors found" % errors) @@ -196,18 +188,21 @@ def _ics_msg_to_message(self, ics_msg): ), is_remote_frame=bool( ics_msg.StatusBitField & ics.SPY_STATUS_REMOTE_FRAME - ) + ), + channel=ics_msg.NetworkID ) def recv(self, timeout=None): - try: + msg = None + if not self.rx_buffer: self._process_msg_queue(timeout=timeout) - ics_msg = self.rx_buffer.get_nowait() - self.rx_buffer.task_done() - return self._ics_msg_to_message(ics_msg) - except queue.Empty: + + try: + ics_msg = self.rx_buffer.popleft() + msg = self._ics_msg_to_message(ics_msg) + except IndexError: pass - return None + return msg def send(self, msg, timeout=None): if not self.opened: From cb231bdda178afe5ad7ea796eb079e8e383691bc Mon Sep 17 00:00:00 2001 From: pierreluctg Date: Fri, 12 Jan 2018 11:31:13 -0500 Subject: [PATCH 19/60] Update CONTRIBUTORS.txt --- CONTRIBUTORS.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CONTRIBUTORS.txt b/CONTRIBUTORS.txt index 8408ccdc8..5a3e183d9 100644 --- a/CONTRIBUTORS.txt +++ b/CONTRIBUTORS.txt @@ -19,3 +19,4 @@ Giuseppe Corbelli Christian Sandberg Eduard Bröcker Boris Wenzlaff +Pierre-Luc Tessier Gagné From ba5d714c99b779d29030da2772ccfe62fc52372c Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sun, 14 Jan 2018 17:26:35 +1100 Subject: [PATCH 20/60] Minor adjustments to slcan --- can/interfaces/slcan.py | 22 +++++++++++++--------- doc/interfaces/slcan.rst | 2 +- doc/interfaces/socketcan_native.rst | 2 +- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/can/interfaces/slcan.py b/can/interfaces/slcan.py index ad2f24197..4f8499028 100755 --- a/can/interfaces/slcan.py +++ b/can/interfaces/slcan.py @@ -1,6 +1,7 @@ """ Interface for slcan compatible interfaces (win32/linux). -(Linux could use slcand/socketcan as well). + +Note Linux users can use slcand/socketcan as well. """ from __future__ import absolute_import @@ -15,6 +16,7 @@ logger = logging.getLogger(__name__) + class slcanBus(BusABC): """ slcan interface @@ -73,7 +75,6 @@ def __init__(self, channel, ttyBaudrate=115200, timeout=1, bitrate=None, **kwarg self.serialPort = io.TextIOWrapper(io.BufferedRWPair(self.serialPortOrig, self.serialPortOrig, 1), newline='\r', line_buffering=True) - # why do we sleep here? time.sleep(self._SLEEP_AFTER_SERIAL_OPEN) if bitrate is not None: @@ -81,7 +82,6 @@ def __init__(self, channel, ttyBaudrate=115200, timeout=1, bitrate=None, **kwarg if bitrate in self._BITRATES: self.write(self._BITRATES[bitrate]) else: - # this only prints the keys of the dict raise ValueError("Invalid bitrate, choose one of " + (', '.join(self._BITRATES)) + '.') self.open() @@ -93,27 +93,31 @@ def recv(self, timeout=None): canId = None remote = False + extended = False frame = [] readStr = self.serialPort.readline() - if not readStr: # if not None and not empty + if not readStr: return None else: - if readStr[0] == 'T': # extended frame + if readStr[0] == 'T': + # extended frame canId = int(readStr[1:9], 16) dlc = int(readStr[9]) extended = True for i in range(0, dlc): frame.append(int(readStr[10 + i * 2:12 + i * 2], 16)) - elif readStr[0] == 't': # normal frame + elif readStr[0] == 't': + # normal frame canId = int(readStr[1:4], 16) dlc = int(readStr[4]) for i in range(0, dlc): frame.append(int(readStr[5 + i * 2:7 + i * 2], 16)) - extended = False - elif readStr[0] == 'r': # remote frame + elif readStr[0] == 'r': + # remote frame canId = int(readStr[1:4], 16) remote = True - elif readStr[0] == 'R': # remote extended frame + elif readStr[0] == 'R': + # remote extended frame canId = int(readStr[1:9], 16) extended = True remote = True diff --git a/doc/interfaces/slcan.rst b/doc/interfaces/slcan.rst index af3a8b565..d47706d51 100755 --- a/doc/interfaces/slcan.rst +++ b/doc/interfaces/slcan.rst @@ -20,4 +20,4 @@ Bus Internals --------- -.. TODO:: Implement and document slcan interface. +.. TODO:: Document internals of slcan interface. diff --git a/doc/interfaces/socketcan_native.rst b/doc/interfaces/socketcan_native.rst index fd9b8711c..f8f4de86b 100644 --- a/doc/interfaces/socketcan_native.rst +++ b/doc/interfaces/socketcan_native.rst @@ -3,7 +3,7 @@ SocketCAN (python) Python 3.3 added support for socketcan for linux systems. -The socketcan_native interface directly uses Python's socket module to +The ``socketcan_native`` interface directly uses Python's socket module to access SocketCAN on linux. This is the most direct route to the kernel and should provide the most responsive one. From c963ce8391a8f4e8cfe9655782ed1ec9dbafb823 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sun, 14 Jan 2018 17:27:38 +1100 Subject: [PATCH 21/60] Minor adjustments to asc logger --- can/io/asc.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/can/io/asc.py b/can/io/asc.py index e40209570..98e391a84 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -8,7 +8,8 @@ CAN_MSG_EXT = 0x80000000 CAN_ID_MASK = 0x1FFFFFFF -logger = logging.getLogger(__name__) +logger = logging.getLogger('can.io.asc') + class ASCReader(object): """ @@ -26,12 +27,12 @@ def _extract_can_id(str_can_id): else: is_extended = False can_id = int(str_can_id, 16) - #logging.debug('ASCReader: _extract_can_id("%s") -> %x, %r', str_can_id, can_id, is_extended) + logging.debug('ASCReader: _extract_can_id("%s") -> %x, %r', str_can_id, can_id, is_extended) return (can_id, is_extended) def __iter__(self): for line in self.file: - #logger.debug("ASCReader: parsing line: '%s'", line.splitlines()[0]) + logger.debug("ASCReader: parsing line: '%s'", line.splitlines()[0]) temp = line.strip() if not temp or not temp[0].isdigit(): From 5d4f0d04e2154e89e7c3925f69fa77dc7ace017f Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sun, 14 Jan 2018 18:58:25 +1100 Subject: [PATCH 22/60] Extend wait period for sql writer testing --- test/logformats_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 39d4024be..d1684042f 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -61,6 +61,7 @@ ) ] + def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=0): """Tests a pair of writer and reader. @@ -98,17 +99,19 @@ class TestCanutilsLog(unittest.TestCase): def test_writer_and_reader(self): _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader) + class TestAscFileFormat(unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" def test_writer_and_reader(self): _test_writer_and_reader(self, can.ASCWriter, can.ASCReader) + class TestSqlFileFormat(unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=0.5) + _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=5) if __name__ == '__main__': From 87b79dbb9d155ea38095a7253addda7421bba6ae Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 19 Dec 2017 08:19:49 +1100 Subject: [PATCH 23/60] fixed .gitignore for the docs build files --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 4ea43b2ae..7666d148a 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,7 @@ coverage.xml *.log # Sphinx documentation -docs/_build/ +doc/_build/ # PyBuilder target/ From 387b586886de31fc9e1419132f36a76b8da1d308 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 15 Jan 2018 20:57:20 +1100 Subject: [PATCH 24/60] Remove unreachable code in bus.__iter__ --- can/bus.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/can/bus.py b/can/bus.py index d6b2526f6..bd7d81a85 100644 --- a/can/bus.py +++ b/can/bus.py @@ -1,4 +1,9 @@ # -*- coding: utf-8 -*- + +""" +Contains the ABC bus implementation. +""" + from __future__ import print_function, absolute_import import abc @@ -104,10 +109,9 @@ def __iter__(self): :yields: :class:`can.Message` msg objects. """ while True: - m = self.recv(timeout=1.0) - if m is not None: - yield m - logger.debug("done iterating over bus messages") + msg = self.recv(timeout=1.0) + if msg is not None: + yield msg def set_filters(self, can_filters=None): """Apply filtering to all messages received by this Bus. From b6ddaf0069235cdf16d8fb0c53de90014f97d9ee Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 15 Jan 2018 03:39:02 +1100 Subject: [PATCH 25/60] Added getting started section to README.rst --- README.rst | 5 +++++ doc/development.rst | 8 ++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 85d128b3d..9c14dc954 100644 --- a/README.rst +++ b/README.rst @@ -46,3 +46,8 @@ questions and answers tagged with ``python+can``. Wherever we interact, we strive to follow the `Python Community Code of Conduct `__. + +Contributing +------------ + +See `doc/development.rst` for getting started. diff --git a/doc/development.rst b/doc/development.rst index 577fb0657..7c794d280 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -12,12 +12,12 @@ https://github.com/hardbyte/python-can Building & Installing --------------------- -This assumes that the commands are executed from the root of the repository. +This assumes that the commands are executed from the root of the repository: -The project can be built and installed with ``python setup.py build`` and +- The project can be built and installed with ``python setup.py build`` and ``python setup.py install``. -The unit tests can be run with ``python setup.py test``. -The docs can be built with ``sphinx-build doc/ doc/_build``. +- The unit tests can be run with ``python setup.py test``. The tests can be run with `python2` and `python3` to check with both major python versions, if they are installed. +- The docs can be built with ``sphinx-build doc/ doc/_build``. Creating a Release From b577d5e939c7bf04ef1c11b379e1a472e9809cc9 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 15 Jan 2018 21:15:42 +1100 Subject: [PATCH 26/60] Update development docs --- doc/development.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc/development.rst b/doc/development.rst index 7c794d280..863213166 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -8,15 +8,19 @@ Contributing Contribute to source code, documentation, examples and report issues: https://github.com/hardbyte/python-can +There is also a `python-can `__ +mailing list for development discussion. + Building & Installing --------------------- -This assumes that the commands are executed from the root of the repository: +The following assumes that the commands are executed from the root of the repository: - The project can be built and installed with ``python setup.py build`` and -``python setup.py install``. -- The unit tests can be run with ``python setup.py test``. The tests can be run with `python2` and `python3` to check with both major python versions, if they are installed. + ``python setup.py install``. +- The unit tests can be run with ``python setup.py test``. The tests can be run with `python2` + and `python3` to check with both major python versions, if they are installed. - The docs can be built with ``sphinx-build doc/ doc/_build``. @@ -54,7 +58,7 @@ The modules in ``python-can`` are: +---------------------------------+------------------------------------------------------+ |:doc:`message ` | Contains the interface independent Message object. | +---------------------------------+------------------------------------------------------+ -|:doc:`notifier ` | An object which can be used to notify listeners. | +|:doc:`io ` | Contains a range of file readers and writers. | +---------------------------------+------------------------------------------------------+ |:doc:`broadcastmanager ` | Contains interface independent broadcast manager | | | code. | From 0d2707361337d1ca6c1b8a278a522f737742d9a7 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 18 Dec 2017 18:01:19 +1100 Subject: [PATCH 27/60] fixed a bug that occurred in ASCReader/ASCWriter when empty messages were sent --- can/io/asc.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/can/io/asc.py b/can/io/asc.py index 98e391a84..d69436cf5 100644 --- a/can/io/asc.py +++ b/can/io/asc.py @@ -37,7 +37,13 @@ def __iter__(self): temp = line.strip() if not temp or not temp[0].isdigit(): continue - (timestamp, channel, dummy) = temp.split(None, 2) # , frameType, dlc, frameData + + try: + (timestamp, channel, dummy) = temp.split(None, 2) # , frameType, dlc, frameData + except ValueError: + # we parsed an empty comment + continue + timestamp = float(timestamp) if dummy.strip()[0:10] == 'ErrorFrame': @@ -109,6 +115,11 @@ def stop(self): def log_event(self, message, timestamp=None): """Add an arbitrary message to the log file.""" + + if not message: # if empty or None + logger.debug("ASCWriter: ignoring empty message") + return + timestamp = (timestamp or time.time()) if timestamp >= self.started: timestamp -= self.started From b5c01f1ce56c739be82ba47f03c47af47e8286ca Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 18 Dec 2017 23:54:26 +1100 Subject: [PATCH 28/60] Fix bug in socketcan_native send raise an can.CanError on timeout with an explicit error message rather than indirectly on failing socket.send call --- can/bus.py | 5 ++- can/interfaces/socketcan/socketcan_native.py | 39 +++++++++++++------- can/notifier.py | 9 +++-- 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/can/bus.py b/can/bus.py index bd7d81a85..84a958b67 100644 --- a/can/bus.py +++ b/can/bus.py @@ -5,11 +5,12 @@ """ from __future__ import print_function, absolute_import - import abc import logging import threading from can.broadcastmanager import ThreadBasedCyclicSendTask + + logger = logging.getLogger(__name__) @@ -22,7 +23,6 @@ class BusABC(object): As well as setting the `channel_info` attribute to a string describing the interface. - """ #: a string describing the underlying bus channel @@ -45,6 +45,7 @@ def __init__(self, channel=None, can_filters=None, **config): :param dict config: Any backend dependent configurations are passed in this dictionary """ + pass @abc.abstractmethod def recv(self, timeout=None): diff --git a/can/interfaces/socketcan/socketcan_native.py b/can/interfaces/socketcan/socketcan_native.py index 1c4280c18..09def076c 100644 --- a/can/interfaces/socketcan/socketcan_native.py +++ b/can/interfaces/socketcan/socketcan_native.py @@ -378,10 +378,12 @@ def __init__(self, channel, receive_own_messages=False, **kwargs): self.socket = createSocket(CAN_RAW) self.channel = channel - # Add any socket options such as can frame filters - if 'can_filters' in kwargs and len(kwargs['can_filters']) > 0: + # add any socket options such as can frame filters + if 'can_filters' in kwargs and kwargs['can_filters']: # = not None or empty log.debug("Creating a filtered can bus") self.set_filters(kwargs['can_filters']) + + # set the receive_own_messages paramater try: self.socket.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_RECV_OWN_MSGS, @@ -396,16 +398,19 @@ def shutdown(self): self.socket.close() def recv(self, timeout=None): - data_ready = True try: if timeout is not None: - data_ready = len(select.select([self.socket], [], [], timeout)[0]) > 0 + # get all sockets that are ready (can be a list with a single value + # being self.socket or an empty list if self.socket is not ready) + ready_receive_sockets, _, _ = select.select([self.socket], [], [], timeout) + else: + ready_receive_sockets = True except OSError: # something bad happened (e.g. the interface went down) log.exception("Error while waiting for timeout") return None - if data_ready: + if ready_receive_sockets: # not empty return captureMessage(self.socket) else: # socket wasn't readable or timeout occurred @@ -423,14 +428,21 @@ def send(self, msg, timeout=None): if msg.is_error_frame: log.warning("Trying to send an error frame - this won't work") arbitration_id |= 0x20000000 - log_tx.debug("Sending: %s", msg) + + logger_tx = log.getChild("tx") + logger_tx.debug("sending: %s", msg) + if timeout: - # Wait for write availability. send will fail below on timeout - select.select([], [self.socket], [], timeout) + # Wait for write availability + _, ready_send_sockets, _ = select.select([], [self.socket], [], timeout) + if not ready_send_sockets: + raise can.CanError("Timeout while sending") + try: bytes_sent = self.socket.send(build_can_frame(arbitration_id, msg.data)) except OSError as exc: raise can.CanError("Transmit failed (%s)" % exc) + if bytes_sent == 0: raise can.CanError("Transmit buffer overflow") @@ -447,8 +459,7 @@ def set_filters(self, can_filters=None): filter_struct = pack_filters(can_filters) self.socket.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, - filter_struct - ) + filter_struct) if __name__ == "__main__": @@ -461,15 +472,15 @@ def set_filters(self, can_filters=None): # ifconfig vcan0 up log.setLevel(logging.DEBUG) - def receiver(e): + def receiver(event): receiver_socket = createSocket() bindSocket(receiver_socket, 'vcan0') print("Receiver is waiting for a message...") - e.set() + event.set() print("Receiver got: ", captureMessage(receiver_socket)) - def sender(e): - e.wait() + def sender(event): + event.wait() sender_socket = createSocket() bindSocket(sender_socket, 'vcan0') sender_socket.send(build_can_frame(0x01, b'\x01\x02\x03')) diff --git a/can/notifier.py b/can/notifier.py index bc9f8f68c..4c2c59604 100644 --- a/can/notifier.py +++ b/can/notifier.py @@ -1,4 +1,7 @@ import threading +import logging + +logger = logging.getLogger('can.Notifier') class Notifier(object): @@ -14,15 +17,15 @@ def __init__(self, bus, listeners, timeout=None): self.listeners = listeners self.bus = bus self.timeout = timeout - #: Exception raised in thread + + # exception raised in thread self.exception = None self.running = threading.Event() self.running.set() - self._reader = threading.Thread(target=self.rx_thread) + self._reader = threading.Thread(target=self.rx_thread, name="can.notifier") self._reader.daemon = True - self._reader.start() def stop(self): From b10528960e5fa166ea8f5d662fab60bdb3c9f3a7 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sun, 17 Dec 2017 20:52:33 +1100 Subject: [PATCH 29/60] Test refactoring * Moved the test messages to a separate module, * added many more and separated them by types, * unified the can.io.* tests further * added many more comments and minor formatting changes * restructured test cases * make tests more reproducible and small fixes --- test/data/__init__.py | 1 + test/data/example_data.py | 129 +++++++++++++++++++++ test/listener_test.py | 154 +++---------------------- test/logformats_test.py | 236 ++++++++++++++++++++++++++------------ 4 files changed, 305 insertions(+), 215 deletions(-) create mode 100644 test/data/__init__.py create mode 100644 test/data/example_data.py diff --git a/test/data/__init__.py b/test/data/__init__.py new file mode 100644 index 000000000..b92cc8726 --- /dev/null +++ b/test/data/__init__.py @@ -0,0 +1 @@ +import example_data diff --git a/test/data/example_data.py b/test/data/example_data.py new file mode 100644 index 000000000..e2288a2c8 --- /dev/null +++ b/test/data/example_data.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +""" +This module contains some example data, like messages fo different +types and example comments with different challenges. +""" + +import random + +from can import Message + + +# make tests more reproducible +# some number that was generated by smashing hands on the keyboard +random.seed(13339115) + + +TEST_TIME = 1483389946.197 # some random number + +# List of messages of different types that can be used in tests +TEST_MESSAGES_BASE = [ + Message( + # empty + ), + Message( + # only data + data=[0x00, 0x42] + ), + Message( + # no data + arbitration_id=0xAB, extended_id=False + ), + Message( + # no data + arbitration_id=0x42, extended_id=True + ), + Message( + # no data + arbitration_id=0xABCDEF, + ), + Message( + # empty data + data=[] + ), + Message( + # empty data + data=[0xFF, 0xFE, 0xFD], + ), + Message( + arbitration_id=0xABCDEF, extended_id=True, + timestamp=TEST_TIME, + data=[1, 2, 3, 4, 5, 6, 7, 8] + ), + Message( + arbitration_id=0x123, extended_id=False, + timestamp=TEST_TIME + 42.42, + data=[0xff, 0xff] + ), + Message( + arbitration_id=0xDADADA, extended_id=True, + timestamp=TEST_TIME + .165, + data=[1, 2, 3, 4, 5, 6, 7, 8] + ), + Message( + arbitration_id=0x123, extended_id=False, + timestamp=TEST_TIME + .365, + data=[254, 255] + ), + Message( + arbitration_id=0x768, extended_id=False, + timestamp=TEST_TIME + 3.165 + ), +] + +TEST_MESSAGES_REMOTE_FRAMES = [ + Message( + arbitration_id=0xDADADA, extended_id=True, is_remote_frame=False, + timestamp=TEST_TIME + .165, + data=[1, 2, 3, 4, 5, 6, 7, 8] + ), + Message( + arbitration_id=0x123, extended_id=False, is_remote_frame=False, + timestamp=TEST_TIME + .365, + data=[254, 255] + ), + Message( + arbitration_id=0x768, extended_id=False, is_remote_frame=True, + timestamp=TEST_TIME + 3.165 + ), + Message( + arbitration_id=0xABCDEF, extended_id=True, is_remote_frame=True, + timestamp=TEST_TIME + 7858.67 + ), +] + +TEST_MESSAGES_ERROR_FRAMES = [ + Message( + is_error_frame=True + ), + Message( + is_error_frame=True, + timestamp=TEST_TIME + 0.170 + ), + Message( + is_error_frame=True, + timestamp=TEST_TIME + 17.157 + ) +] + +TEST_COMMENTS = [ + "This is the first comment", + "", # empty comment + "This third comment contains some strange characters: 'ä\"§$%&/()=?__::_Öüßêè and ends here.", + ( + "This fourth comment is quite long! " \ + "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. Lorem ipsum dolor sit amet, consetetur sadipscing elitr, sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet. " \ + "Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi. Lorem ipsum dolor sit amet, consectetuer adipiscing elit, sed diam nonummy nibh euismod tincidunt ut laoreet dolore magna aliquam erat volutpat. " \ + "Ut wisi enim ad minim veniam, quis nostrud exerci tation ullamcorper suscipit lobortis nisl ut aliquip ex ea commodo consequat. Duis autem vel eum iriure dolor in hendrerit in vulputate velit esse molestie consequat, vel illum dolore eu feugiat nulla facilisis at vero eros et accumsan et iusto odio dignissim qui blandit praesent luptatum zzril delenit augue duis dolore te feugait nulla facilisi." \ + ), +] + +def generate_message(arbitration_id): + """ + Generates a new message with the given ID, some random data + and a non-extended ID. + """ + data = [random.randrange(0, 2 ** 8 - 1) for _ in range(8)] + msg = Message(arbitration_id=arbitration_id, data=data, extended_id=False) + return msg diff --git a/test/listener_test.py b/test/listener_test.py index 2c3c71661..c3732daea 100755 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -8,45 +8,27 @@ import can +from data.example_data import generate_message + channel = 'vcan0' can.rc['interface'] = 'virtual' -logging.getLogger("").setLevel(logging.DEBUG) - - -# List of messages of different types that can be used in tests -TEST_MESSAGES = [ - can.Message( - arbitration_id=0xDADADA, extended_id=True, is_remote_frame=False, - timestamp=1483389464.165, - data=[1, 2, 3, 4, 5, 6, 7, 8]), - can.Message( - arbitration_id=0x123, extended_id=False, is_remote_frame=False, - timestamp=1483389464.365, - data=[254, 255]), - can.Message( - arbitration_id=0x768, extended_id=False, is_remote_frame=True, - timestamp=1483389466.165), - can.Message(is_error_frame=True, timestamp=1483389466.170), -] - - -def generate_message(arbitration_id): - data = [random.randrange(0, 2 ** 8 - 1) for _ in range(8)] - m = can.Message(arbitration_id=arbitration_id, data=data, extended_id=False) - return m +logging.getLogger('').setLevel(logging.DEBUG) +# make tests more reproducible +# some number that was generated by smashing hands on the keyboard +random.seed(13339115) class ListenerImportTest(unittest.TestCase): def testClassesImportable(self): - assert hasattr(can, 'Listener') - assert hasattr(can, 'BufferedReader') - assert hasattr(can, 'Notifier') - assert hasattr(can, 'ASCWriter') - assert hasattr(can, 'CanutilsLogWriter') - assert hasattr(can, 'SqlReader') - + self.assertTrue(hasattr(can, 'Listener')) + self.assertTrue(hasattr(can, 'BufferedReader')) + self.assertTrue(hasattr(can, 'Notifier')) + self.assertTrue(hasattr(can, 'ASCWriter')) + self.assertTrue(hasattr(can, 'CanutilsLogWriter')) + self.assertTrue(hasattr(can, 'SqlReader')) + # TODO add more? class BusTest(unittest.TestCase): @@ -75,7 +57,7 @@ def test_filetype_to_instance(extension, klass): test_filetype_to_instance('log', can.CanutilsLogWriter) test_filetype_to_instance("blf", can.BLFWriter) test_filetype_to_instance("csv", can.CSVWriter) - test_filetype_to_instance("db", can.SqliteWriter) + test_filetype_to_instance("db", can.SqliteWriter) test_filetype_to_instance("txt", can.Printer) def testBufferedListenerReceives(self): @@ -84,114 +66,6 @@ def testBufferedListenerReceives(self): m = a_listener.get_message(0.2) self.assertIsNotNone(m) - def testSQLWriterReceives(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - a_listener = can.SqliteWriter(f.name) - a_listener(generate_message(0xDADADA)) - # Small delay so we don't stop before we actually block trying to read - sleep(0.5) - a_listener.stop() - - con = sqlite3.connect(f.name) - c = con.cursor() - c.execute("select * from messages") - msg = c.fetchone() - con.close() - self.assertEqual(msg[1], 0xDADADA) - - def testSQLWriterWritesToSameFile(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - - first_listener = can.SqliteWriter(f.name) - first_listener(generate_message(0x01)) - - sleep(first_listener.MAX_TIME_BETWEEN_WRITES) - first_listener.stop() - - second_listener = can.SqliteWriter(f.name) - second_listener(generate_message(0x02)) - - sleep(second_listener.MAX_TIME_BETWEEN_WRITES) - - second_listener.stop() - - con = sqlite3.connect(f.name) - - with con: - c = con.cursor() - - c.execute("select COUNT() from messages") - self.assertEqual(2, c.fetchone()[0]) - - c.execute("select * from messages") - msg1 = c.fetchone() - msg2 = c.fetchone() - - assert msg1[1] == 0x01 - assert msg2[1] == 0x02 - - - def testAscListener(self): - a_listener = can.ASCWriter("test.asc", channel=2) - a_listener.log_event("This is some comment") - msg = can.Message(extended_id=True, - timestamp=a_listener.started + 0.5, - arbitration_id=0xabcdef, - data=[1, 2, 3, 4, 5, 6, 7, 8]) - a_listener(msg) - msg = can.Message(extended_id=False, - timestamp=a_listener.started + 1, - arbitration_id=0x123, - data=[0xff, 0xff]) - a_listener(msg) - msg = can.Message(extended_id=True, - timestamp=a_listener.started + 1.5, - is_remote_frame=True, - dlc=8, - arbitration_id=0xabcdef) - a_listener(msg) - msg = can.Message(is_error_frame=True, - timestamp=a_listener.started + 1.6, - arbitration_id=0xabcdef) - a_listener(msg) - a_listener.stop() - with open("test.asc", "r") as f: - output_contents = f.read() - - self.assertTrue('This is some comment' in output_contents) - - -class BLFTest(unittest.TestCase): - - def test_reader(self): - logfile = os.path.join(os.path.dirname(__file__), "data", "logfile.blf") - messages = list(can.BLFReader(logfile)) - self.assertEqual(len(messages), 1) - self.assertEqual(messages[0], - can.Message( - extended_id=False, - arbitration_id=0x64, - data=[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8])) - - def test_reader_writer(self): - f = tempfile.NamedTemporaryFile('w', delete=False) - f.close() - filename = f.name - - writer = can.BLFWriter(filename) - for msg in TEST_MESSAGES: - writer(msg) - writer.log_event("One comment which should be attached to last message") - writer.log_event("Another comment", TEST_MESSAGES[-1].timestamp + 2) - writer.stop() - - messages = list(can.BLFReader(filename)) - self.assertEqual(len(messages), len(TEST_MESSAGES)) - for msg1, msg2 in zip(messages, TEST_MESSAGES): - self.assertEqual(msg1, msg2) - self.assertAlmostEqual(msg1.timestamp, msg2.timestamp) if __name__ == '__main__': unittest.main() diff --git a/test/logformats_test.py b/test/logformats_test.py index d1684042f..feaa4cdf2 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -1,89 +1,113 @@ +""" +This test module test theseparatent reader/writer combinations of the can.io.* +modules by writing some messages to a temporary file and reading it again. +Then it checks if the messages that ware read are same ones as the +ones that were written. It also checks that the order of the messages +is correct. The types of messages that are tested differs between the +different writer/reader pairs and one some, comments are inserted and read +again. +""" + import unittest import tempfile from time import sleep +import sqlite3 +import os + +try: + # Python 3 + from itertools import zip_longest +except ImportError: + # Python 2 + from itertools import izip_longest as zip_longest import can -TIME = 1483389946.197 # some random number - -# List of messages of different types that can be used in tests -TEST_MESSAGES = [ - can.Message( - # empty - ), - can.Message( - # only data - data=[0x00, 0x42] - ), - can.Message( - # no data - arbitration_id=0xAB, extended_id=False - ), - can.Message( - # no data - arbitration_id=0x42, extended_id=True - ), - can.Message( - # empty data - data=[] - ), - can.Message( - arbitration_id=0xDADADA, extended_id=True, is_remote_frame=False, - timestamp=TIME + .165, - data=[1, 2, 3, 4, 5, 6, 7, 8] - ), - can.Message( - arbitration_id=0x123, extended_id=False, is_remote_frame=False, - timestamp=TIME + .365, - data=[254, 255] - ), - can.Message( - arbitration_id=0x768, extended_id=False, is_remote_frame=True, - timestamp=TIME + 3.165 - ), - can.Message( - is_error_frame=True, - timestamp=TIME + 0.170 - ), - can.Message( - arbitration_id=0xABCDEF, extended_id=True, - timestamp=TIME, - data=[1, 2, 3, 4, 5, 6, 7, 8] - ), - can.Message( - arbitration_id=0x123, extended_id=False, - timestamp=TIME + 42.42, - data=[0xff, 0xff] - ), - can.Message( - arbitration_id=0xABCDEF, extended_id=True, is_remote_frame=True, - timestamp=TIME + 7858.67 - ) -] - - -def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=0): - """Tests a pair of writer and reader. - - The :attr:`sleep_time` specifies the time to sleep after - writing all messages. +from data.example_data import TEST_MESSAGES_BASE, TEST_MESSAGES_REMOTE_FRAMES, \ + TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS, \ + generate_message + +def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=None, + check_remote_frames=True, check_error_frames=True, + check_comments=False): + """Tests a pair of writer and reader by writing all data first and + then reading all data and checking if they could be reconstructed + correctly. + + :param test_case: the test case the use the assert methods on + :param sleep_time: specifies the time to sleep after writing all messages. + gets ignored when set to None + :param check_remote_frames: if true, also tests remote frames + :param check_error_frames: if true, also tests error frames + :param check_comments: if true, also inserts comments at some + locations and checks if they are contained anywhere literally + in the resulting file. The locations as selected randomly + but deterministically, which makes the test reproducible. """ + assert isinstance(test_case, unittest.TestCase), \ + "test_case has to be a subclass of unittest.TestCase" + + if check_comments: + # we check this because of the lack of a common base class + # we filter for not starts with '__' so we do not get all the builtin + # methods when logging to the console + test_case.assertIn('log_event', [d for d in dir(writer_constructor) if not d.startswith('__')], + "cannot check comments with this writer: {}".format(writer_constructor)) + + # create a temporary file temp = tempfile.NamedTemporaryFile('w', delete=False) temp.close() filename = temp.name - writer = writer_constructor(filename) - for msg in TEST_MESSAGES: - writer(msg) + # get all test messages + original_messages = TEST_MESSAGES_BASE + if check_remote_frames: + original_messages += TEST_MESSAGES_REMOTE_FRAMES + if check_error_frames: + original_messages += TEST_MESSAGES_ERROR_FRAMES + + # get all test comments + original_comments = TEST_COMMENTS - sleep(sleep_time) + # create writer + writer = writer_constructor(filename) + + # write + if check_comments: + # write messages and insert comments here and there + # Note: we make no assumptions about the length of original_messages and original_comments + for msg, comment in zip_longest(original_messages, original_comments, fillvalue=None): + # msg and comment might be None + if comment is not None: + print("writing comment: ", comment) + writer.log_event(comment) # we already know that this method exists + print("writing comment: ", comment) + if msg is not None: + print("writing message: ", msg) + writer(msg) + print("writing message: ", msg) + else: + # ony write messages + for msg in original_messages: + print("writing message: ", msg) + writer(msg) + print("writing message: ", msg) + + # sleep and close the writer + if sleep_time is not None: + sleep(sleep_time) writer.stop() - messages = list(reader_constructor(filename)) - test_case.assertEqual(len(messages), len(TEST_MESSAGES)) + # read all written messages + read_messages = list(reader_constructor(filename)) + + # check if at least the number of messages matches + test_case.assertEqual(len(read_messages), len(original_messages), + "the number of written messages does not match the number of read messages") - for i, (read, original) in enumerate(zip(messages, TEST_MESSAGES)): + # check the order and content of the individual messages + for i, (read, original) in enumerate(zip(read_messages, original_messages)): try: test_case.assertEqual(read, original) test_case.assertAlmostEqual(read.timestamp, original.timestamp) @@ -92,27 +116,89 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s exception.args += ("test failed at index #{}".format(i), ) raise exception + # check if the comments are contained in the file + if check_comments: + # read the entire outout file + with open(filename, 'r') as file: + output_contents = file.read() + # check each, if they can be found in there literally + for comment in original_comments: + test_case.assertTrue(comment in output_contents) + class TestCanutilsLog(unittest.TestCase): """Tests can.CanutilsLogWriter and can.CanutilsLogReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader) - + _test_writer_and_reader(self, can.CanutilsLogWriter, can.CanutilsLogReader, + check_error_frames=False, # TODO this should get fixed, see Issue #217 + check_comments=False) class TestAscFileFormat(unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.ASCWriter, can.ASCReader) - + _test_writer_and_reader(self, can.ASCWriter, can.ASCReader, + check_error_frames=False, # TODO this should get fixed, see Issue #218 + check_comments=True) class TestSqlFileFormat(unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, sleep_time=5) + _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, + sleep_time=0.5, + check_comments=False) + + def testSQLWriterWritesToSameFile(self): + f = tempfile.NamedTemporaryFile('w', delete=False) + f.close() + first_listener = can.SqliteWriter(f.name) + first_listener(generate_message(0x01)) + + sleep(first_listener.MAX_TIME_BETWEEN_WRITES) + first_listener.stop() + + second_listener = can.SqliteWriter(f.name) + second_listener(generate_message(0x02)) + + sleep(second_listener.MAX_TIME_BETWEEN_WRITES) + + second_listener.stop() + + con = sqlite3.connect(f.name) + + with con: + c = con.cursor() + + c.execute("select COUNT() from messages") + self.assertEqual(2, c.fetchone()[0]) + + c.execute("select * from messages") + msg1 = c.fetchone() + msg2 = c.fetchone() + + self.assertEqual(msg1[1], 0x01) + self.assertEqual(msg2[1], 0x02) + +class TestBlfFileFormat(unittest.TestCase): + """Tests can.BLFWriter and can.BLFReader""" + + def test_writer_and_reader(self): + _test_writer_and_reader(self, can.BLFWriter, can.BLFReader, + sleep_time=None, + check_comments=False) + + def test_reader(self): + logfile = os.path.join(os.path.dirname(__file__), "data", "logfile.blf") + messages = list(can.BLFReader(logfile)) + self.assertEqual(len(messages), 1) + self.assertEqual(messages[0], + can.Message( + extended_id=False, + arbitration_id=0x64, + data=[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8])) if __name__ == '__main__': unittest.main() From 60039b9c98e05ff57b7ce0f2fd98654a83358683 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 19 Dec 2017 08:22:08 +1100 Subject: [PATCH 30/60] name refactoring in socketcan_native --- can/interfaces/socketcan/socketcan_native.py | 44 ++++++++++---------- doc/interfaces/socketcan_native.rst | 18 ++++---- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/can/interfaces/socketcan/socketcan_native.py b/can/interfaces/socketcan/socketcan_native.py index 09def076c..917c55f49 100644 --- a/can/interfaces/socketcan/socketcan_native.py +++ b/can/interfaces/socketcan/socketcan_native.py @@ -136,7 +136,7 @@ def create_bcm_socket(channel): return s -def send_bcm(socket, data): +def send_bcm(bcm_socket, data): """ Send raw frame to a BCM socket and handle errors. @@ -145,21 +145,21 @@ def send_bcm(socket, data): :return: """ try: - return socket.send(data) + return bcm_socket.send(data) except OSError as e: base = "Couldn't send CAN BCM frame. OS Error {}: {}\n".format(e.errno, os.strerror(e.errno)) if e.errno == errno.EINVAL: - raise can.CanError( - base + "You are probably referring to a non-existing frame.") + raise can.CanError(base + "You are probably referring to a non-existing frame.") + elif e.errno == errno.ENETDOWN: - raise can.CanError( - base + "The CAN interface appears to be down." - ) + raise can.CanError(base + "The CAN interface appears to be down.") + elif e.errno == errno.EBADF: raise can.CanError(base + "The CAN socket appears to be closed.") + else: - raise + raise e def _add_flags_to_can_id(message): can_id = message.arbitration_id @@ -184,7 +184,8 @@ def __init__(self, channel, *args, **kwargs): super(SocketCanBCMBase, self).__init__(*args, **kwargs) -class CyclicSendTask(SocketCanBCMBase, LimitedDurationCyclicSendTaskABC, ModifiableCyclicTaskABC, RestartableCyclicTaskABC): +class CyclicSendTask(SocketCanBCMBase, LimitedDurationCyclicSendTaskABC, + ModifiableCyclicTaskABC, RestartableCyclicTaskABC): """ A socketcan cyclic send task supports: @@ -196,7 +197,6 @@ class CyclicSendTask(SocketCanBCMBase, LimitedDurationCyclicSendTaskABC, Modifia def __init__(self, channel, message, period): """ - :param channel: The name of the CAN channel to connect to. :param message: The message to be sent periodically. :param period: The rate in seconds at which to send the message. @@ -259,7 +259,7 @@ def __init__(self, channel, message, count, initial_period, subsequent_period): send_bcm(self.bcm_socket, header + frame) -def createSocket(can_protocol=None): +def create_socket(can_protocol=None): """Creates a CAN socket. The socket can be BCM or RAW. The socket will be returned unbound to any interface. @@ -286,7 +286,7 @@ def createSocket(can_protocol=None): return sock -def bindSocket(sock, channel='can0'): +def bind_socket(sock, channel='can0'): """ Binds the given socket to the given interface. @@ -300,7 +300,7 @@ def bindSocket(sock, channel='can0'): log.debug('Bound socket.') -def captureMessage(sock): +def capture_message(sock): """ Captures a message from given socket. @@ -375,7 +375,7 @@ def __init__(self, channel, receive_own_messages=False, **kwargs): :param list can_filters: A list of dictionaries, each containing a "can_id" and a "can_mask". """ - self.socket = createSocket(CAN_RAW) + self.socket = create_socket(CAN_RAW) self.channel = channel # add any socket options such as can frame filters @@ -388,10 +388,10 @@ def __init__(self, channel, receive_own_messages=False, **kwargs): self.socket.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_RECV_OWN_MSGS, struct.pack('i', receive_own_messages)) - except Exception as e: + except socket.error as e: log.error("Could not receive own messages (%s)", e) - bindSocket(self.socket, channel) + bind_socket(self.socket, channel) super(SocketcanNative_Bus, self).__init__() def shutdown(self): @@ -411,7 +411,7 @@ def recv(self, timeout=None): return None if ready_receive_sockets: # not empty - return captureMessage(self.socket) + return capture_message(self.socket) else: # socket wasn't readable or timeout occurred return None @@ -473,16 +473,16 @@ def set_filters(self, can_filters=None): log.setLevel(logging.DEBUG) def receiver(event): - receiver_socket = createSocket() - bindSocket(receiver_socket, 'vcan0') + receiver_socket = create_socket() + bind_socket(receiver_socket, 'vcan0') print("Receiver is waiting for a message...") event.set() - print("Receiver got: ", captureMessage(receiver_socket)) + print("Receiver got: ", capture_message(receiver_socket)) def sender(event): event.wait() - sender_socket = createSocket() - bindSocket(sender_socket, 'vcan0') + sender_socket = create_socket() + bind_socket(sender_socket, 'vcan0') sender_socket.send(build_can_frame(0x01, b'\x01\x02\x03')) print("Sender sent a message.") diff --git a/doc/interfaces/socketcan_native.rst b/doc/interfaces/socketcan_native.rst index f8f4de86b..cb6f9aea4 100644 --- a/doc/interfaces/socketcan_native.rst +++ b/doc/interfaces/socketcan_native.rst @@ -28,19 +28,19 @@ Bus Internals --------- -createSocket -~~~~~~~~~~~~ +create_socket +~~~~~~~~~~~~~ -.. autofunction:: can.interfaces.socketcan.socketcan_native.createSocket +.. autofunction:: can.interfaces.socketcan.socketcan_native.create_socket -bindSocket -~~~~~~~~~~ +bind_socket +~~~~~~~~~~~ -.. autofunction:: can.interfaces.socketcan.socketcan_native.bindSocket +.. autofunction:: can.interfaces.socketcan.socketcan_native.bind_socket -captureMessage -~~~~~~~~~~~~~~ +capture_message +~~~~~~~~~~~~~~~ -.. autofunction:: can.interfaces.socketcan.socketcan_native.captureMessage +.. autofunction:: can.interfaces.socketcan.socketcan_native.capture_message From 1661cf38795eb8445669fea91b2d551a2bcfdfe4 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 15 Jan 2018 03:19:37 +1100 Subject: [PATCH 31/60] changed to relative imports fixed import of test/data/example_data.py in the test cases --- test/data/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/data/__init__.py b/test/data/__init__.py index b92cc8726..e69de29bb 100644 --- a/test/data/__init__.py +++ b/test/data/__init__.py @@ -1 +0,0 @@ -import example_data From cfc431a0a91495d4970c4cb3781cbcd27e4a4b75 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sat, 16 Dec 2017 22:40:18 +1100 Subject: [PATCH 32/60] added more documentation for the message object (cherry picked from commit d96efc4) Signed-off-by: Brian Thorne --- can/io/blf.py | 1 + can/message.py | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/can/io/blf.py b/can/io/blf.py index 10355efb1..0e20e0491 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -10,6 +10,7 @@ of uncompressed data each. This data contains the actual CAN messages and other objects types. """ + import struct import zlib import datetime diff --git a/can/message.py b/can/message.py index c566317e5..09905ac55 100644 --- a/can/message.py +++ b/can/message.py @@ -4,9 +4,22 @@ class Message(object): """ - The :class:`~can.Message` object is used to represent CAN messages for both sending and receiving. + The :class:`~can.Message` object is used to represent CAN messages for + both sending and receiving. + + Messages can use extended identifiers, be remote or error frames, contain + data and can be associated to a channel. + + When testing for equality of the messages, the timestamp and the channel + is not used for comparing. + + .. note:: + + This class does not strictly check the input. Thus, the caller must + prevent the creation of invalid messages. Possible problems include + the `dlc` field not matching the length of `data` or creating a message + with both `is_remote_frame` and `is_error_frame` set to True. - Messages can use extended identifiers, be remote or error frames, and contain data. """ def __init__(self, timestamp=0.0, is_remote_frame=False, extended_id=True, @@ -100,7 +113,7 @@ def __repr__(self): def __eq__(self, other): return (isinstance(other, self.__class__) and self.arbitration_id == other.arbitration_id and - #self.timestamp == other.timestamp and # TODO: explain this + #self.timestamp == other.timestamp and # allow the timestamp to differ self.id_type == other.id_type and self.dlc == other.dlc and self.data == other.data and From 02ebeb578cea48feb1f09825f2c085b58cd6eb31 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 15 Jan 2018 21:52:32 +1100 Subject: [PATCH 33/60] Update logformat test docs --- test/logformats_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index feaa4cdf2..492943c5b 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -1,11 +1,11 @@ """ -This test module test theseparatent reader/writer combinations of the can.io.* +This test module test the separate reader/writer combinations of the can.io.* modules by writing some messages to a temporary file and reading it again. -Then it checks if the messages that ware read are same ones as the +Then it checks if the messages that were read are same ones as the ones that were written. It also checks that the order of the messages is correct. The types of messages that are tested differs between the -different writer/reader pairs and one some, comments are inserted and read -again. +different writer/reader pairs - e.g., some don't handle error frames and +comments. """ import unittest @@ -27,6 +27,7 @@ TEST_MESSAGES_ERROR_FRAMES, TEST_COMMENTS, \ generate_message + def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, sleep_time=None, check_remote_frames=True, check_error_frames=True, check_comments=False): @@ -182,6 +183,7 @@ def testSQLWriterWritesToSameFile(self): self.assertEqual(msg1[1], 0x01) self.assertEqual(msg2[1], 0x02) + class TestBlfFileFormat(unittest.TestCase): """Tests can.BLFWriter and can.BLFReader""" From 31c9e2de1584ec06e67812466036bae7346e7a80 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 15 Jan 2018 22:14:49 +1100 Subject: [PATCH 34/60] Add missing io imports to listener test --- can/io/log.py | 1 + can/io/logger.py | 1 + test/data/example_data.py | 8 ++++---- test/listener_test.py | 22 ++++++++++++++++++++-- 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/can/io/log.py b/can/io/log.py index b18ab005a..3a7816e56 100644 --- a/can/io/log.py +++ b/can/io/log.py @@ -8,6 +8,7 @@ CAN_ERR_BUSERROR = 0x00000080 CAN_ERR_DLC = 8 + class CanutilsLogReader(object): """ Iterator of CAN messages from a .log Logging File (candump -L). diff --git a/can/io/logger.py b/can/io/logger.py index 6fe5a005c..450f33c1c 100755 --- a/can/io/logger.py +++ b/can/io/logger.py @@ -5,6 +5,7 @@ from .stdout import Printer from .log import CanutilsLogWriter + class Logger(object): """ Logs CAN messages to a file. diff --git a/test/data/example_data.py b/test/data/example_data.py index e2288a2c8..bc097f755 100644 --- a/test/data/example_data.py +++ b/test/data/example_data.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """ -This module contains some example data, like messages fo different +This module contains some example data, like messages of different types and example comments with different challenges. """ @@ -11,11 +11,10 @@ # make tests more reproducible -# some number that was generated by smashing hands on the keyboard random.seed(13339115) - -TEST_TIME = 1483389946.197 # some random number +# some random number +TEST_TIME = 1483389946.197 # List of messages of different types that can be used in tests TEST_MESSAGES_BASE = [ @@ -119,6 +118,7 @@ ), ] + def generate_message(arbitration_id): """ Generates a new message with the given ID, some random data diff --git a/test/listener_test.py b/test/listener_test.py index c3732daea..40c3ed97c 100755 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -16,19 +16,37 @@ logging.getLogger('').setLevel(logging.DEBUG) # make tests more reproducible -# some number that was generated by smashing hands on the keyboard random.seed(13339115) + class ListenerImportTest(unittest.TestCase): def testClassesImportable(self): self.assertTrue(hasattr(can, 'Listener')) self.assertTrue(hasattr(can, 'BufferedReader')) self.assertTrue(hasattr(can, 'Notifier')) + self.assertTrue(hasattr(can, 'Logger')) + self.assertTrue(hasattr(can, 'ASCWriter')) + self.assertTrue(hasattr(can, 'ASCReader')) + + self.assertTrue(hasattr(can, 'BLFReader')) + self.assertTrue(hasattr(can, 'BLFWriter')) + + self.assertTrue(hasattr(can, 'CSVWriter')) + self.assertTrue(hasattr(can, 'CanutilsLogWriter')) + self.assertTrue(hasattr(can, 'CanutilsLogReader')) + self.assertTrue(hasattr(can, 'SqlReader')) - # TODO add more? + self.assertTrue(hasattr(can, 'SqliteWriter')) + + self.assertTrue(hasattr(can, 'Printer')) + + self.assertTrue(hasattr(can, 'LogReader')) + + self.assertTrue(hasattr(can.io.player, 'MessageSync')) + class BusTest(unittest.TestCase): From 3d23c87f39f3a0aecda60f4031beef38663556f4 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Mon, 15 Jan 2018 22:28:13 +1100 Subject: [PATCH 35/60] Add white space and more test time to logformats_test --- test/logformats_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/logformats_test.py b/test/logformats_test.py index 492943c5b..4bb5fba4f 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -98,6 +98,7 @@ def _test_writer_and_reader(test_case, writer_constructor, reader_constructor, s # sleep and close the writer if sleep_time is not None: sleep(sleep_time) + writer.stop() # read all written messages @@ -135,6 +136,7 @@ def test_writer_and_reader(self): check_error_frames=False, # TODO this should get fixed, see Issue #217 check_comments=False) + class TestAscFileFormat(unittest.TestCase): """Tests can.ASCWriter and can.ASCReader""" @@ -143,12 +145,13 @@ def test_writer_and_reader(self): check_error_frames=False, # TODO this should get fixed, see Issue #218 check_comments=True) + class TestSqlFileFormat(unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" def test_writer_and_reader(self): _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, - sleep_time=0.5, + sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES, check_comments=False) def testSQLWriterWritesToSameFile(self): @@ -202,5 +205,6 @@ def test_reader(self): arbitration_id=0x64, data=[0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8])) + if __name__ == '__main__': unittest.main() From 1b85a333fac7d421fdc6e4179111b4276207c8d5 Mon Sep 17 00:00:00 2001 From: Christian Sandberg Date: Thu, 30 Nov 2017 20:26:28 +0100 Subject: [PATCH 36/60] Add CAN-FD support Initial support for socketcan_native and kvaser Experimental support for BLF logger Only warn on DLC too long since some loggers support long data lengths but not FD frames --- can/interfaces/ixxat/canlib.py | 15 ++- can/interfaces/kvaser/canlib.py | 91 ++++++++++++---- can/interfaces/kvaser/constants.py | 12 +++ .../socketcan/socketcan_constants.py | 6 ++ can/interfaces/socketcan/socketcan_ctypes.py | 4 +- can/interfaces/socketcan/socketcan_native.py | 102 ++++++++++++------ can/io/blf.py | 49 +++++++-- can/message.py | 21 +++- doc/message.rst | 25 +++++ test/back2back_test.py | 32 +++++- test/listener_test.py | 5 + 11 files changed, 286 insertions(+), 76 deletions(-) diff --git a/can/interfaces/ixxat/canlib.py b/can/interfaces/ixxat/canlib.py index 03c697401..1b098200a 100644 --- a/can/interfaces/ixxat/canlib.py +++ b/can/interfaces/ixxat/canlib.py @@ -442,14 +442,13 @@ def recv(self, timeout=None): # The _message.dwTime is a 32bit tick value and will overrun, # so expect to see the value restarting from 0 rx_msg = Message( - self._message.dwTime / self._tick_resolution, # Relative time in s - True if self._message.uMsgInfo.Bits.rtr else False, - True if self._message.uMsgInfo.Bits.ext else False, - False, - self._message.dwMsgId, - self._message.uMsgInfo.Bits.dlc, - self._message.abData[:self._message.uMsgInfo.Bits.dlc], - self.channel + timestamp=self._message.dwTime / self._tick_resolution, # Relative time in s + is_remote_frame=True if self._message.uMsgInfo.Bits.rtr else False, + extended_id=True if self._message.uMsgInfo.Bits.ext else False, + arbitration_id=self._message.dwMsgId, + dlc=self._message.uMsgInfo.Bits.dlc, + data=self._message.abData[:self._message.uMsgInfo.Bits.dlc], + channel=self.channel ) log.debug('Recv()ed message %s', rx_msg) diff --git a/can/interfaces/kvaser/canlib.py b/can/interfaces/kvaser/canlib.py index 7da22addb..f430356e7 100644 --- a/can/interfaces/kvaser/canlib.py +++ b/can/interfaces/kvaser/canlib.py @@ -169,6 +169,12 @@ def __check_bus_handle_validity(handle, function, arguments): restype=canstat.c_canStatus, errcheck=__check_status) + canSetBusParamsFd = __get_canlib_function("canSetBusParamsFd", + argtypes=[c_canHandle, ctypes.c_long, + ctypes.c_uint, ctypes.c_uint, + ctypes.c_uint], + restype=canstat.c_canStatus, + errcheck=__check_status) canSetBusOutputControl = __get_canlib_function("canSetBusOutputControl", argtypes=[c_canHandle, @@ -254,15 +260,25 @@ def init_kvaser_library(): DRIVER_MODE_NORMAL = True -BITRATE_OBJS = {1000000 : canstat.canBITRATE_1M, - 500000 : canstat.canBITRATE_500K, - 250000 : canstat.canBITRATE_250K, - 125000 : canstat.canBITRATE_125K, - 100000 : canstat.canBITRATE_100K, - 83000 : canstat.canBITRATE_83K, - 62000 : canstat.canBITRATE_62K, - 50000 : canstat.canBITRATE_50K, - 10000 : canstat.canBITRATE_10K} +BITRATE_OBJS = { + 1000000: canstat.canBITRATE_1M, + 500000: canstat.canBITRATE_500K, + 250000: canstat.canBITRATE_250K, + 125000: canstat.canBITRATE_125K, + 100000: canstat.canBITRATE_100K, + 83000: canstat.canBITRATE_83K, + 62000: canstat.canBITRATE_62K, + 50000: canstat.canBITRATE_50K, + 10000: canstat.canBITRATE_10K +} + +BITRATE_FD = { + 500000: canstat.canFD_BITRATE_500K_80P, + 1000000: canstat.canFD_BITRATE_1M_80P, + 2000000: canstat.canFD_BITRATE_2M_80P, + 4000000: canstat.canFD_BITRATE_4M_80P, + 8000000: canstat.canFD_BITRATE_8M_60P +} class KvaserBus(BusABC): @@ -285,6 +301,8 @@ def __init__(self, channel, can_filters=None, **config): :param int bitrate: Bitrate of channel in bit/s + :param bool accept_virtual: + If virtual channels should be accepted. :param int tseg1: Time segment 1, that is, the number of quanta from (but not including) the Sync Segment to the sampling point. @@ -313,6 +331,11 @@ def __init__(self, channel, can_filters=None, **config): Only works if single_handle is also False. If you want to receive messages from other applications on the same computer, set this to True or set single_handle to True. + :param bool fd: + If CAN-FD frames should be supported. + :param int data_bitrate: + Which bitrate to use for data phase in CAN FD. + Defaults to arbitration bitrate. """ log.info("CAN Filters: {}".format(can_filters)) log.info("Got configuration of: {}".format(config)) @@ -324,6 +347,9 @@ def __init__(self, channel, can_filters=None, **config): driver_mode = config.get('driver_mode', DRIVER_MODE_NORMAL) single_handle = config.get('single_handle', False) receive_own_messages = config.get('receive_own_messages', False) + accept_virtual = config.get('accept_virtual', True) + fd = config.get('fd', False) + data_bitrate = config.get('data_bitrate', None) try: channel = int(channel) @@ -331,9 +357,6 @@ def __init__(self, channel, can_filters=None, **config): raise ValueError('channel must be an integer') self.channel = channel - if 'tseg1' not in config and bitrate in BITRATE_OBJS: - bitrate = BITRATE_OBJS[bitrate] - log.debug('Initialising bus instance') self.single_handle = single_handle @@ -348,12 +371,33 @@ def __init__(self, channel, can_filters=None, **config): if idx == channel: self.channel_info = channel_info + flags = 0 + if accept_virtual: + flags |= canstat.canOPEN_ACCEPT_VIRTUAL + if fd: + flags |= canstat.canOPEN_CAN_FD + log.debug('Creating read handle to bus channel: %s' % channel) - self._read_handle = canOpenChannel(channel, canstat.canOPEN_ACCEPT_VIRTUAL) + self._read_handle = canOpenChannel(channel, flags) canIoCtl(self._read_handle, canstat.canIOCTL_SET_TIMER_SCALE, ctypes.byref(ctypes.c_long(TIMESTAMP_RESOLUTION)), 4) + + if fd: + if 'tseg1' not in config and bitrate in BITRATE_FD: + # Use predefined bitrate for arbitration + bitrate = BITRATE_FD[bitrate] + if data_bitrate in BITRATE_FD: + # Use predefined bitrate for data + data_bitrate = BITRATE_FD[data_bitrate] + elif not data_bitrate: + # Use same bitrate for arbitration and data phase + data_bitrate = bitrate + canSetBusParamsFd(self._read_handle, bitrate, tseg1, tseg2, sjw) + else: + if 'tseg1' not in config and bitrate in BITRATE_OBJS: + bitrate = BITRATE_OBJS[bitrate] canSetBusParams(self._read_handle, bitrate, tseg1, tseg2, sjw, no_samp, 0) # By default, use local echo if single handle is used (see #160) @@ -370,7 +414,7 @@ def __init__(self, channel, can_filters=None, **config): self._write_handle = self._read_handle else: log.debug('Creating separate handle for TX on channel: %s' % channel) - self._write_handle = canOpenChannel(channel, canstat.canOPEN_ACCEPT_VIRTUAL) + self._write_handle = canOpenChannel(channel, flags) canBusOn(self._read_handle) self.set_filters(can_filters) @@ -437,7 +481,7 @@ def recv(self, timeout=None): Read a message from kvaser device. """ arb_id = ctypes.c_long(0) - data = ctypes.create_string_buffer(8) + data = ctypes.create_string_buffer(64) dlc = ctypes.c_uint(0) flags = ctypes.c_uint(0) timestamp = ctypes.c_ulong(0) @@ -467,6 +511,9 @@ def recv(self, timeout=None): is_extended = bool(flags & canstat.canMSG_EXT) is_remote_frame = bool(flags & canstat.canMSG_RTR) is_error_frame = bool(flags & canstat.canMSG_ERROR_FRAME) + is_fd = bool(flags & canstat.canFDMSG_FDF) + bitrate_switch = bool(flags & canstat.canFDMSG_BRS) + error_state_indicator = bool(flags & canstat.canFDMSG_ESI) msg_timestamp = timestamp.value * TIMESTAMP_FACTOR rx_msg = Message(arbitration_id=arb_id.value, data=data_array[:dlc.value], @@ -474,6 +521,9 @@ def recv(self, timeout=None): extended_id=is_extended, is_error_frame=is_error_frame, is_remote_frame=is_remote_frame, + is_fd=is_fd, + bitrate_switch=bitrate_switch, + error_state_indicator=error_state_indicator, channel=self.channel, timestamp=msg_timestamp + self._timestamp_offset) rx_msg.flags = flags @@ -491,6 +541,10 @@ def send(self, msg, timeout=None): flags |= canstat.canMSG_RTR if msg.is_error_frame: flags |= canstat.canMSG_ERROR_FRAME + if msg.is_fd: + flags |= canstat.canFDMSG_FDF + if msg.bitrate_switch: + flags |= canstat.canFDMSG_BRS ArrayConstructor = ctypes.c_byte * msg.dlc buf = ArrayConstructor(*msg.data) canWrite(self._write_handle, @@ -520,9 +574,10 @@ def shutdown(self): # Wait for transmit queue to be cleared try: canWriteSync(self._write_handle, 100) - except CANLIBError as e: - log.warning("There may be messages in the transmit queue that could " - "not be transmitted before going bus off (%s)", e) + except CANLIBError: + # Not a huge deal and it seems that we get timeout if no messages + # exists in the buffer at all + pass if not self.single_handle: canBusOff(self._read_handle) canClose(self._read_handle) diff --git a/can/interfaces/kvaser/constants.py b/can/interfaces/kvaser/constants.py index b6a7dce8a..20ca5204e 100644 --- a/can/interfaces/kvaser/constants.py +++ b/can/interfaces/kvaser/constants.py @@ -61,6 +61,10 @@ def CANSTATUS_SUCCESS(status): canMSG_TXACK = 0x0040 canMSG_TXRQ = 0x0080 +canFDMSG_FDF = 0x010000 +canFDMSG_BRS = 0x020000 +canFDMSG_ESI = 0x040000 + canMSGERR_MASK = 0xff00 canMSGERR_HW_OVERRUN = 0x0200 canMSGERR_SW_OVERRUN = 0x0400 @@ -159,6 +163,8 @@ def CANSTATUS_SUCCESS(status): canOPEN_REQUIRE_INIT_ACCESS = 0x0080 canOPEN_NO_INIT_ACCESS = 0x0100 canOPEN_ACCEPT_LARGE_DLC = 0x0200 +canOPEN_CAN_FD = 0x0400 +canOPEN_CAN_FD_NONISO = 0x0800 canIOCTL_GET_RX_BUFFER_LEVEL = 8 canIOCTL_GET_TX_BUFFER_LEVEL = 9 @@ -230,3 +236,9 @@ def CANSTATUS_SUCCESS(status): canBITRATE_50K = -7 canBITRATE_83K = -8 canBITRATE_10K = -9 + +canFD_BITRATE_500K_80P = -1000 +canFD_BITRATE_1M_80P = -1001 +canFD_BITRATE_2M_80P = -1002 +canFD_BITRATE_4M_80P = -1003 +canFD_BITRATE_8M_60P = -1004 diff --git a/can/interfaces/socketcan/socketcan_constants.py b/can/interfaces/socketcan/socketcan_constants.py index a1b92e148..b3a7447fa 100644 --- a/can/interfaces/socketcan/socketcan_constants.py +++ b/can/interfaces/socketcan/socketcan_constants.py @@ -30,6 +30,7 @@ RX_ANNOUNCE_RESUME = 0x0100 TX_RESET_MULTI_IDX = 0x0200 RX_RTR_FRAME = 0x0400 +CAN_FD_FRAME = 0x0800 CAN_RAW = 1 CAN_BCM = 2 @@ -58,6 +59,11 @@ SKT_ERRFLG = 0x0001 SKT_RTRFLG = 0x0002 +CANFD_BRS = 0x01 +CANFD_ESI = 0x02 + +CANFD_MTU = 72 + PYCAN_ERRFLG = 0x0020 PYCAN_STDFLG = 0x0002 PYCAN_RTRFLG = 0x0001 diff --git a/can/interfaces/socketcan/socketcan_ctypes.py b/can/interfaces/socketcan/socketcan_ctypes.py index d1f64384d..785c14aac 100644 --- a/can/interfaces/socketcan/socketcan_ctypes.py +++ b/can/interfaces/socketcan/socketcan_ctypes.py @@ -346,9 +346,9 @@ def _build_can_frame(message): # TODO need to understand the extended frame format frame = CAN_FRAME() frame.can_id = arbitration_id - frame.can_dlc = len(message.data) + frame.can_dlc = message.dlc - frame.data[0:frame.can_dlc] = message.data + frame.data[0:len(message.data)] = message.data log.debug("sizeof frame: %d", ctypes.sizeof(frame)) return frame diff --git a/can/interfaces/socketcan/socketcan_native.py b/can/interfaces/socketcan/socketcan_native.py index 917c55f49..10933d79e 100644 --- a/can/interfaces/socketcan/socketcan_native.py +++ b/can/interfaces/socketcan/socketcan_native.py @@ -44,11 +44,10 @@ # The 32bit can id is directly followed by the 8bit data link count # The data field is aligned on an 8 byte boundary, hence we add padding # which aligns the data field to an 8 byte boundary. -can_frame_fmt = "=IB3x8s" -can_frame_size = struct.calcsize(can_frame_fmt) +CAN_FRAME_HEADER_STRUCT = struct.Struct("=IBB2x") -def build_can_frame(can_id, data): +def build_can_frame(msg): """ CAN frame packing/unpacking (see 'struct can_frame' in ) /** * struct can_frame - basic CAN frame structure @@ -61,10 +60,34 @@ def build_can_frame(can_id, data): __u8 can_dlc; /* data length code: 0 .. 8 */ __u8 data[8] __attribute__((aligned(8))); }; + + /** + * struct canfd_frame - CAN flexible data rate frame structure + * @can_id: CAN ID of the frame and CAN_*_FLAG flags, see canid_t definition + * @len: frame payload length in byte (0 .. CANFD_MAX_DLEN) + * @flags: additional flags for CAN FD + * @__res0: reserved / padding + * @__res1: reserved / padding + * @data: CAN FD frame payload (up to CANFD_MAX_DLEN byte) + */ + struct canfd_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 len; /* frame payload length in byte */ + __u8 flags; /* additional flags for CAN FD */ + __u8 __res0; /* reserved / padding */ + __u8 __res1; /* reserved / padding */ + __u8 data[CANFD_MAX_DLEN] __attribute__((aligned(8))); + }; """ - can_dlc = len(data) - data = data.ljust(8, b'\x00') - return struct.pack(can_frame_fmt, can_id, can_dlc, data) + can_id = _add_flags_to_can_id(msg) + flags = 0 + if msg.bitrate_switch: + flags |= CANFD_BRS + if msg.error_state_indicator: + flags |= CANFD_ESI + max_len = 64 if msg.is_fd else 8 + data = msg.data.ljust(max_len, b'\x00') + return CAN_FRAME_HEADER_STRUCT.pack(can_id, msg.dlc, flags) + data def build_bcm_header(opcode, flags, count, ival1_seconds, ival1_usec, ival2_seconds, ival2_usec, can_id, nframes): @@ -90,15 +113,16 @@ def build_bcm_header(opcode, flags, count, ival1_seconds, ival1_usec, ival2_seco nframes) -def build_bcm_tx_delete_header(can_id): +def build_bcm_tx_delete_header(can_id, flags): opcode = CAN_BCM_TX_DELETE - return build_bcm_header(opcode, 0, 0, 0, 0, 0, 0, can_id, 1) + return build_bcm_header(opcode, flags, 0, 0, 0, 0, 0, can_id, 1) -def build_bcm_transmit_header(can_id, count, initial_period, subsequent_period): +def build_bcm_transmit_header(can_id, count, initial_period, subsequent_period, + msg_flags): opcode = CAN_BCM_TX_SETUP - flags = SETTIMER | STARTTIMER + flags = msg_flags | SETTIMER | STARTTIMER if initial_period > 0: # Note `TX_COUNTEVT` creates the message TX_EXPIRED when count expires @@ -118,8 +142,11 @@ def split_time(value): def dissect_can_frame(frame): - can_id, can_dlc, data = struct.unpack(can_frame_fmt, frame) - return can_id, can_dlc, data[:can_dlc] + can_id, can_dlc, flags = CAN_FRAME_HEADER_STRUCT.unpack_from(frame) + if len(frame) != CANFD_MTU: + # Flags not valid in non-FD frames + flags = 0 + return can_id, can_dlc, flags, frame[8:8+can_dlc] def create_bcm_socket(channel): @@ -208,8 +235,10 @@ def __init__(self, channel, message, period): def _tx_setup(self, message): # Create a low level packed frame to pass to the kernel self.can_id_with_flags = _add_flags_to_can_id(message) - header = build_bcm_transmit_header(self.can_id_with_flags, 0, 0.0, self.period) - frame = build_can_frame(self.can_id_with_flags, message.data) + self.flags = CAN_FD_FRAME if message.is_fd else 0 + header = build_bcm_transmit_header(self.can_id_with_flags, 0, 0.0, + self.period, self.flags) + frame = build_can_frame(message) log.debug("Sending BCM command") send_bcm(self.bcm_socket, header + frame) @@ -222,7 +251,7 @@ def stop(self): """ log.debug("Stopping periodic task") - stopframe = build_bcm_tx_delete_header(self.can_id_with_flags) + stopframe = build_bcm_tx_delete_header(self.can_id_with_flags, self.flags) send_bcm(self.bcm_socket, stopframe) def modify_data(self, message): @@ -248,12 +277,13 @@ def __init__(self, channel, message, count, initial_period, subsequent_period): super(MultiRateCyclicSendTask, self).__init__(channel, message, subsequent_period) # Create a low level packed frame to pass to the kernel - frame = build_can_frame(self.can_id, message.data) + frame = build_can_frame(message) header = build_bcm_transmit_header( - self.can_id, + self.can_id_with_flags, count, initial_period, - subsequent_period) + subsequent_period, + self.flags) log.info("Sending BCM TX_SETUP command") send_bcm(self.bcm_socket, header + frame) @@ -311,7 +341,7 @@ def capture_message(sock): """ # Fetching the Arb ID, DLC and Data try: - cf, addr = sock.recvfrom(can_frame_size) + cf, addr = sock.recvfrom(CANFD_MTU) except BlockingIOError: log.debug('Captured no data, socket in non-blocking mode.') return None @@ -323,7 +353,8 @@ def capture_message(sock): log.exception("Captured no data.") return None - can_id, can_dlc, data = dissect_can_frame(cf) + can_id, can_dlc, flags, data = dissect_can_frame(cf) + log.debug('Received: can_id=%x, can_dlc=%x, data=%s', can_id, can_dlc, data) # Fetching the timestamp binary_structure = "@LL" @@ -340,6 +371,9 @@ def capture_message(sock): is_extended_frame_format = bool(can_id & 0x80000000) is_remote_transmission_request = bool(can_id & 0x40000000) is_error_frame = bool(can_id & 0x20000000) + is_fd = len(cf) == CANFD_MTU + bitrate_switch = bool(flags & CANFD_BRS) + error_state_indicator = bool(flags & CANFD_ESI) if is_extended_frame_format: log.debug("CAN: Extended") @@ -354,6 +388,9 @@ def capture_message(sock): extended_id=is_extended_frame_format, is_remote_frame=is_remote_transmission_request, is_error_frame=is_error_frame, + is_fd=is_fd, + bitrate_switch=bitrate_switch, + error_state_indicator=error_state_indicator, dlc=can_dlc, data=data) @@ -363,20 +400,22 @@ def capture_message(sock): class SocketcanNative_Bus(BusABC): - channel_info = "native socketcan channel" - def __init__(self, channel, receive_own_messages=False, **kwargs): + def __init__(self, channel, receive_own_messages=False, fd=False, **kwargs): """ :param str channel: The can interface name with which to create this bus. An example channel would be 'vcan0'. :param bool receive_own_messages: If messages transmitted should also be received back. + :param bool fd: + If CAN-FD frames should be supported. :param list can_filters: A list of dictionaries, each containing a "can_id" and a "can_mask". """ self.socket = create_socket(CAN_RAW) self.channel = channel + self.channel_info = "native socketcan channel '%s'" % channel # add any socket options such as can frame filters if 'can_filters' in kwargs and kwargs['can_filters']: # = not None or empty @@ -391,6 +430,11 @@ def __init__(self, channel, receive_own_messages=False, **kwargs): except socket.error as e: log.error("Could not receive own messages (%s)", e) + if fd: + self.socket.setsockopt(socket.SOL_CAN_RAW, + socket.CAN_RAW_FD_FRAMES, + struct.pack('i', 1)) + bind_socket(self.socket, channel) super(SocketcanNative_Bus, self).__init__() @@ -418,20 +462,8 @@ def recv(self, timeout=None): def send(self, msg, timeout=None): log.debug("We've been asked to write a message to the bus") - arbitration_id = msg.arbitration_id - if msg.id_type: - log.debug("sending an extended id type message") - arbitration_id |= 0x80000000 - if msg.is_remote_frame: - log.debug("requesting a remote frame") - arbitration_id |= 0x40000000 - if msg.is_error_frame: - log.warning("Trying to send an error frame - this won't work") - arbitration_id |= 0x20000000 - logger_tx = log.getChild("tx") logger_tx.debug("sending: %s", msg) - if timeout: # Wait for write availability _, ready_send_sockets, _ = select.select([], [self.socket], [], timeout) @@ -439,7 +471,7 @@ def send(self, msg, timeout=None): raise can.CanError("Timeout while sending") try: - bytes_sent = self.socket.send(build_can_frame(arbitration_id, msg.data)) + bytes_sent = self.socket.send(build_can_frame(msg)) except OSError as exc: raise can.CanError("Transmit failed (%s)" % exc) diff --git a/can/io/blf.py b/can/io/blf.py index 0e20e0491..bdd994d17 100644 --- a/can/io/blf.py +++ b/can/io/blf.py @@ -38,6 +38,10 @@ # channel, flags, dlc, arbitration id, data CAN_MSG_STRUCT = struct.Struct(" 64: + logger.warning("data link count was %d but it should be less than or equal to 64", self.dlc) + if not is_fd and self.dlc > 8: + logger.warning("data link count was %d but it should be less than or equal to 8", self.dlc) def __str__(self): field_strings = ["Timestamp: {0:15.6f}".format(self.timestamp)] @@ -66,6 +74,7 @@ def __str__(self): "X" if self.id_type else "S", "E" if self.is_error_frame else " ", "R" if self.is_remote_frame else " ", + "F" if self.is_fd else " ", ]) field_strings.append(flag_string) @@ -108,6 +117,10 @@ def __repr__(self): "data=[{}]".format(", ".join(data))] if self.channel is not None: args.append("channel={}".format(self.channel)) + if self.is_fd: + args.append("is_fd=True") + args.append("bitrate_switch={}".format(self.bitrate_switch)) + args.append("error_state_indicator={}".format(self.error_state_indicator)) return "can.Message({})".format(", ".join(args)) def __eq__(self, other): @@ -118,7 +131,9 @@ def __eq__(self, other): self.dlc == other.dlc and self.data == other.data and self.is_remote_frame == other.is_remote_frame and - self.is_error_frame == other.is_error_frame) + self.is_error_frame == other.is_error_frame and + self.is_fd == other.is_fd and + self.bitrate_switch == other.bitrate_switch) def __hash__(self): return hash(( @@ -127,6 +142,8 @@ def __hash__(self): self.id_type, self.dlc, self.data, + self.is_fd, + self.bitrate_switch, self.is_remote_frame, self.is_error_frame )) diff --git a/doc/message.rst b/doc/message.rst index 7e2bb881e..cd350d9f1 100644 --- a/doc/message.rst +++ b/doc/message.rst @@ -66,6 +66,9 @@ Message The :abbr:`DLC (Data Link Count)` parameter of a CAN message is an integer between 0 and 8 representing the frame payload length. + In the case of a CAN FD message, this indicates the data length in + number of bytes. + >>> m = Message(data=[1, 2, 3]) >>> m.dlc 3 @@ -116,6 +119,28 @@ Message Timestamp: 0.000000 ID: 00000000 X R DLC: 0 + .. attribute:: is_fd + + :type: bool + + Indicates that this message is a CAN FD message. + + + .. attribute:: bitrate_switch + + :type: bool + + If this is a CAN FD message, this indicates that a higher bitrate + was used for the data transmission. + + + .. attribute:: error_state_indicator + + :type: bool + + If this is a CAN FD message, this indicates an error active state. + + .. attribute:: timestamp :type: float diff --git a/test/back2back_test.py b/test/back2back_test.py index 146815516..c20aac5e2 100644 --- a/test/back2back_test.py +++ b/test/back2back_test.py @@ -6,11 +6,12 @@ BITRATE = 500000 TIMEOUT = 0.1 +TEST_CAN_FD = True INTERFACE_1 = 'virtual' -CHANNEL_1 = 0 +CHANNEL_1 = 'vcan0' INTERFACE_2 = 'virtual' -CHANNEL_2 = 0 +CHANNEL_2 = 'vcan0' class Back2BackTestCase(unittest.TestCase): @@ -22,10 +23,14 @@ class Back2BackTestCase(unittest.TestCase): def setUp(self): self.bus1 = can.interface.Bus(channel=CHANNEL_1, bustype=INTERFACE_1, - bitrate=BITRATE) + bitrate=BITRATE, + fd=TEST_CAN_FD, + single_handle=True) self.bus2 = can.interface.Bus(channel=CHANNEL_2, bustype=INTERFACE_2, - bitrate=BITRATE) + bitrate=BITRATE, + fd=TEST_CAN_FD, + single_handle=True) def tearDown(self): self.bus1.shutdown() @@ -38,6 +43,8 @@ def _check_received_message(self, recv_msg, sent_msg): self.assertEqual(recv_msg.id_type, sent_msg.id_type) self.assertEqual(recv_msg.is_remote_frame, sent_msg.is_remote_frame) self.assertEqual(recv_msg.is_error_frame, sent_msg.is_error_frame) + self.assertEqual(recv_msg.is_fd, sent_msg.is_fd) + self.assertEqual(recv_msg.bitrate_switch, sent_msg.bitrate_switch) self.assertEqual(recv_msg.dlc, sent_msg.dlc) if not sent_msg.is_remote_frame: self.assertSequenceEqual(recv_msg.data, sent_msg.data) @@ -94,6 +101,23 @@ def test_dlc_less_than_eight(self): data=[4, 5, 6]) self._send_and_receive(msg) + @unittest.skipUnless(TEST_CAN_FD, "Don't test CAN-FD") + def test_fd_message(self): + msg = can.Message(is_fd=True, + extended_id=True, + arbitration_id=0x56789, + data=[0xff] * 64) + self._send_and_receive(msg) + + @unittest.skipUnless(TEST_CAN_FD, "Don't test CAN-FD") + def test_fd_message_with_brs(self): + msg = can.Message(is_fd=True, + bitrate_switch=True, + extended_id=True, + arbitration_id=0x98765, + data=[0xff] * 48) + self._send_and_receive(msg) + if __name__ == '__main__': unittest.main() diff --git a/test/listener_test.py b/test/listener_test.py index 2c3c71661..22eeed561 100755 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -28,6 +28,11 @@ arbitration_id=0x768, extended_id=False, is_remote_frame=True, timestamp=1483389466.165), can.Message(is_error_frame=True, timestamp=1483389466.170), + can.Message( + is_fd=True, bitrate_switch=True, + arbitration_id=0x123456, extended_id=True, + data=[0xff] * 64, + timestamp=1483389466.365), ] From 816ee40b38e3eca07eae0cc17f5f67e61c496763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Wed, 17 Jan 2018 09:31:29 -0500 Subject: [PATCH 37/60] Fix neovi serial number compare to allow string ot int format --- can/interfaces/ics_neovi/neovi_bus.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/can/interfaces/ics_neovi/neovi_bus.py b/can/interfaces/ics_neovi/neovi_bus.py index 1a4b2a455..c8d8b1374 100644 --- a/can/interfaces/ics_neovi/neovi_bus.py +++ b/can/interfaces/ics_neovi/neovi_bus.py @@ -100,10 +100,7 @@ def _open_device(self, type_filter=None, serial=None): devices = ics.find_devices() for device in devices: - if serial is None: - dev = device - break - if str(device.SerialNumber) == serial: + if serial is None or str(device.SerialNumber) == str(serial): dev = device break else: From 839fbf24ba89c1067ed5be034d1a66b5071dce18 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 15 Jan 2018 13:00:58 +0100 Subject: [PATCH 38/60] Update development.rst --- doc/development.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/development.rst b/doc/development.rst index 863213166..6fb0de2f4 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -19,8 +19,8 @@ The following assumes that the commands are executed from the root of the reposi - The project can be built and installed with ``python setup.py build`` and ``python setup.py install``. -- The unit tests can be run with ``python setup.py test``. The tests can be run with `python2` - and `python3` to check with both major python versions, if they are installed. +- The unit tests can be run with ``python setup.py test``. The tests can be run with ``python2``, + ``python3``, ``pypy`` or ``pypy3`` to test with other python versions, if they are installed. - The docs can be built with ``sphinx-build doc/ doc/_build``. From 1538449a28a8d75f89805d12df20de9cd226f05d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Wed, 17 Jan 2018 10:54:22 -0500 Subject: [PATCH 39/60] Adding string representation to CAN Bus Abstract Base Class Adding __str__ to CAN Bus Abstract Base Class for a nicely printable string representation of an the Bus using channel_info. --- can/bus.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/can/bus.py b/can/bus.py index 84a958b67..f42a4a149 100644 --- a/can/bus.py +++ b/can/bus.py @@ -47,6 +47,9 @@ def __init__(self, channel=None, can_filters=None, **config): """ pass + def __str__(self): + return self.channel_info + @abc.abstractmethod def recv(self, timeout=None): """Block waiting for a message from the Bus. From 14742de705a4439daef1544ca56284a62d0490b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Tue, 9 Jan 2018 11:01:46 -0500 Subject: [PATCH 40/60] Using pluggy to allow plug-able can interfaces. Allows user of python-can to create their own plug-able interface without the need to modify the python-can source. --- can/interface.py | 18 +++++++++++++++++- can/plugin.py | 13 +++++++++++++ setup.py | 1 + 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 can/plugin.py diff --git a/can/interface.py b/can/interface.py index 1ac7954b1..bd6b010d6 100644 --- a/can/interface.py +++ b/can/interface.py @@ -1,9 +1,12 @@ from __future__ import absolute_import +from pluggy import HookimplMarker + import can import importlib from can.broadcastmanager import CyclicSendTaskABC, MultiRateCyclicSendTaskABC +from can.plugin import get_pluginmanager from can.util import load_config # interface_name => (module, classname) @@ -24,6 +27,17 @@ } +hookimpl = HookimplMarker('pythoncan') + + +@hookimpl(trylast=True) +def pythoncan_interface(interface): + """This hook is used to process the initial config + and possibly input arguments. + """ + return BACKENDS.get(interface) + + class Bus(object): """ Instantiates a CAN Bus of the given `bustype`, falls back to reading a @@ -58,7 +72,9 @@ def __new__(cls, other, channel=None, *args, **kwargs): # Import the correct Bus backend try: - (module_name, class_name) = BACKENDS[interface] + interfaces_hook = get_pluginmanager().hook + (module_name, class_name) = interfaces_hook.pythoncan_interface( + interface=interface)[0] except KeyError: raise NotImplementedError("CAN interface '{}' not supported".format(interface)) diff --git a/can/plugin.py b/can/plugin.py new file mode 100644 index 000000000..a0aa656ec --- /dev/null +++ b/can/plugin.py @@ -0,0 +1,13 @@ +from pluggy import PluginManager + +from can.interfaces import hookspecs + + +def get_pluginmanager(load_entrypoints=True): + pm = PluginManager("pythoncan") + pm.add_hookspecs(hookspecs) + # XXX load internal plugins here + if load_entrypoints: + pm.load_setuptools_entrypoints("python_can.interface") + pm.check_pending() + return pm diff --git a/setup.py b/setup.py index 8813466ed..b6dd21f03 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ # Tests can be run using `python setup.py test` test_suite="nose.collector", tests_require=['mock', 'nose', 'pyserial'], + install_requires=['pluggy'], extras_require={ 'serial': ['pyserial'], 'neovi': ['python-ics'], From a69670cfb6afe0c975e24cf14281345bd0005f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Mon, 15 Jan 2018 08:33:25 -0500 Subject: [PATCH 41/60] Adding missing hookspecs --- can/interfaces/hookspecs.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 can/interfaces/hookspecs.py diff --git a/can/interfaces/hookspecs.py b/can/interfaces/hookspecs.py new file mode 100644 index 000000000..9ebceb9c6 --- /dev/null +++ b/can/interfaces/hookspecs.py @@ -0,0 +1,11 @@ +from pluggy import HookspecMarker + + +hookspec = HookspecMarker("pythoncan") + + +@hookspec +def pythoncan_interface(interface): + """ return tuple (module, classname) containing python-can interface info + for the interface if supported by this implementation + """ From 5f9b1fa287477a1435226ad74854ae99a2e5b63a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 18 Jan 2018 09:29:40 -0500 Subject: [PATCH 42/60] Remove the usage of plugin for internal interfaces --- can/interface.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/can/interface.py b/can/interface.py index bd6b010d6..597707fd1 100644 --- a/can/interface.py +++ b/can/interface.py @@ -27,17 +27,6 @@ } -hookimpl = HookimplMarker('pythoncan') - - -@hookimpl(trylast=True) -def pythoncan_interface(interface): - """This hook is used to process the initial config - and possibly input arguments. - """ - return BACKENDS.get(interface) - - class Bus(object): """ Instantiates a CAN Bus of the given `bustype`, falls back to reading a @@ -73,8 +62,11 @@ def __new__(cls, other, channel=None, *args, **kwargs): # Import the correct Bus backend try: interfaces_hook = get_pluginmanager().hook - (module_name, class_name) = interfaces_hook.pythoncan_interface( - interface=interface)[0] + plugin = interfaces_hook.pythoncan_interface(interface=interface) + if plugin: + (module_name, class_name) = plugin[0] + else: + (module_name, class_name) = BACKENDS[interface] except KeyError: raise NotImplementedError("CAN interface '{}' not supported".format(interface)) From d64f87ff8d58bcb032382e50fb0de758cab7bbaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 18 Jan 2018 09:34:10 -0500 Subject: [PATCH 43/60] removed unused import --- can/interface.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/can/interface.py b/can/interface.py index 597707fd1..965037702 100644 --- a/can/interface.py +++ b/can/interface.py @@ -1,7 +1,5 @@ from __future__ import absolute_import -from pluggy import HookimplMarker - import can import importlib From 878e04314416d6fb04db7c029976c628e429bd34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 18 Jan 2018 10:50:31 -0500 Subject: [PATCH 44/60] Simplifying the plugin interface --- can/interface.py | 15 ++++++++------- can/interfaces/__init__.py | 6 ++++++ can/interfaces/hookspecs.py | 11 ----------- can/plugin.py | 13 ------------- setup.py | 1 - 5 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 can/interfaces/hookspecs.py delete mode 100644 can/plugin.py diff --git a/can/interface.py b/can/interface.py index 965037702..174f03b58 100644 --- a/can/interface.py +++ b/can/interface.py @@ -4,7 +4,7 @@ import importlib from can.broadcastmanager import CyclicSendTaskABC, MultiRateCyclicSendTaskABC -from can.plugin import get_pluginmanager +from pkg_resources import iter_entry_points from can.util import load_config # interface_name => (module, classname) @@ -25,6 +25,12 @@ } +BACKENDS.update({ + interface.name: (interface.module_name, interface.attrs[0]) + for interface in iter_entry_points('python_can.interface') +}) + + class Bus(object): """ Instantiates a CAN Bus of the given `bustype`, falls back to reading a @@ -59,12 +65,7 @@ def __new__(cls, other, channel=None, *args, **kwargs): # Import the correct Bus backend try: - interfaces_hook = get_pluginmanager().hook - plugin = interfaces_hook.pythoncan_interface(interface=interface) - if plugin: - (module_name, class_name) = plugin[0] - else: - (module_name, class_name) = BACKENDS[interface] + (module_name, class_name) = BACKENDS[interface] except KeyError: raise NotImplementedError("CAN interface '{}' not supported".format(interface)) diff --git a/can/interfaces/__init__.py b/can/interfaces/__init__.py index ae1685c0d..1bd132731 100644 --- a/can/interfaces/__init__.py +++ b/can/interfaces/__init__.py @@ -2,8 +2,14 @@ """ Interfaces contain low level implementations that interact with CAN hardware. """ +from pkg_resources import iter_entry_points VALID_INTERFACES = set(['kvaser', 'serial', 'pcan', 'socketcan_native', 'socketcan_ctypes', 'socketcan', 'usb2can', 'ixxat', 'nican', 'iscan', 'vector', 'virtual', 'neovi', 'slcan']) + + +VALID_INTERFACES.update(set([ + interface.name for interface in iter_entry_points('python_can.interface') +])) diff --git a/can/interfaces/hookspecs.py b/can/interfaces/hookspecs.py deleted file mode 100644 index 9ebceb9c6..000000000 --- a/can/interfaces/hookspecs.py +++ /dev/null @@ -1,11 +0,0 @@ -from pluggy import HookspecMarker - - -hookspec = HookspecMarker("pythoncan") - - -@hookspec -def pythoncan_interface(interface): - """ return tuple (module, classname) containing python-can interface info - for the interface if supported by this implementation - """ diff --git a/can/plugin.py b/can/plugin.py deleted file mode 100644 index a0aa656ec..000000000 --- a/can/plugin.py +++ /dev/null @@ -1,13 +0,0 @@ -from pluggy import PluginManager - -from can.interfaces import hookspecs - - -def get_pluginmanager(load_entrypoints=True): - pm = PluginManager("pythoncan") - pm.add_hookspecs(hookspecs) - # XXX load internal plugins here - if load_entrypoints: - pm.load_setuptools_entrypoints("python_can.interface") - pm.check_pending() - return pm diff --git a/setup.py b/setup.py index b6dd21f03..8813466ed 100644 --- a/setup.py +++ b/setup.py @@ -29,7 +29,6 @@ # Tests can be run using `python setup.py test` test_suite="nose.collector", tests_require=['mock', 'nose', 'pyserial'], - install_requires=['pluggy'], extras_require={ 'serial': ['pyserial'], 'neovi': ['python-ics'], From 94f4238feec4c93815d21f5fb214ea45c96d973a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 18 Jan 2018 11:05:57 -0500 Subject: [PATCH 45/60] Documenting the plugin interface entry point --- doc/interfaces.rst | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/doc/interfaces.rst b/doc/interfaces.rst index b150f6cfb..24e7ab63c 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -24,6 +24,18 @@ The available interfaces are: interfaces/vector interfaces/virtual +Additional interfaces can be added via a plugin interface. An external package +can register a new interface by using the `python_can.interface` entry point. + +The format of the entry point is `interface_name=module:classname`. + +:: + + entry_points={ + 'python_can.interface': [ + "interface_name=module:classname", + ] + }, The *Interface Names* are listed in :doc:`configuration`. From 05819158ad7c46f62ed1c3b052ba2b2a3b58aee3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 18 Jan 2018 11:11:22 -0500 Subject: [PATCH 46/60] Minor doc change --- doc/interfaces.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/interfaces.rst b/doc/interfaces.rst index 24e7ab63c..f0a87ff6b 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -27,7 +27,8 @@ The available interfaces are: Additional interfaces can be added via a plugin interface. An external package can register a new interface by using the `python_can.interface` entry point. -The format of the entry point is `interface_name=module:classname`. +The format of the entry point is `interface_name=module:classname` where +`classname` is a can.BusABC implementation. :: From 6735a0ea753b70c19cf098fbb131daffcb1258df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Thu, 18 Jan 2018 11:14:02 -0500 Subject: [PATCH 47/60] Minor doc change --- doc/interfaces.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/interfaces.rst b/doc/interfaces.rst index f0a87ff6b..00d1da37d 100644 --- a/doc/interfaces.rst +++ b/doc/interfaces.rst @@ -25,10 +25,10 @@ The available interfaces are: interfaces/virtual Additional interfaces can be added via a plugin interface. An external package -can register a new interface by using the `python_can.interface` entry point. +can register a new interface by using the ``python_can.interface`` entry point. -The format of the entry point is `interface_name=module:classname` where -`classname` is a can.BusABC implementation. +The format of the entry point is ``interface_name=module:classname`` where +``classname`` is a :class:`can.BusABC` concrete implementation. :: From 1d708a0b57032d35243e2dc1ab4ced3d41dab810 Mon Sep 17 00:00:00 2001 From: Mathias Giacomuzzi Date: Wed, 31 Jan 2018 16:56:56 +0100 Subject: [PATCH 48/60] add interface example definition for beginners --- examples/send_one.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/examples/send_one.py b/examples/send_one.py index fc5d7949b..46ae20980 100755 --- a/examples/send_one.py +++ b/examples/send_one.py @@ -3,7 +3,10 @@ def send_one(): - bus = can.interface.Bus() + bus = can.interface.Bus(bustype='pcan', channel='PCAN_USBBUS1', bitrate=250000) + #bus = can.interface.Bus(bustype='ixxat', channel=0, bitrate=250000) + #bus = can.interface.Bus(bustype='vector', app_name='CANalyzer', channel=0, bitrate=250000) + msg = can.Message(arbitration_id=0xc0ffee, data=[0, 25, 0, 1, 3, 1, 4, 1], extended_id=True) From 486f776c02c7733df69e1f449baed79d41ba1b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pierre-Luc=20Tessier=20Gagn=C3=A9?= Date: Tue, 6 Feb 2018 11:11:08 -0500 Subject: [PATCH 49/60] Adding ICSApiError class and error handling --- can/interfaces/ics_neovi/neovi_bus.py | 44 ++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 4 deletions(-) diff --git a/can/interfaces/ics_neovi/neovi_bus.py b/can/interfaces/ics_neovi/neovi_bus.py index c8d8b1374..6402d59f8 100644 --- a/can/interfaces/ics_neovi/neovi_bus.py +++ b/can/interfaces/ics_neovi/neovi_bus.py @@ -11,7 +11,7 @@ import logging from collections import deque -from can import Message +from can import Message, CanError from can.bus import BusABC logger = logging.getLogger(__name__) @@ -26,6 +26,35 @@ ics = None +class ICSApiError(CanError): + # A critical error which affects operation or accuracy. + ICS_SPY_ERR_CRITICAL = 0x10 + # An error which is not understood. + ICS_SPY_ERR_QUESTION = 0x20 + # An important error which may be critical depending on the application + ICS_SPY_ERR_EXCLAMATION = 0x30 + # An error which probably does not need attention. + ICS_SPY_ERR_INFORMATION = 0x40 + + def __init__( + self, error_number, description_short, description_long, + severity, restart_needed + ): + super(ICSApiError, self).__init__(description_short) + self.error_number = error_number + self.description_short = description_short + self.description_long = description_long + self.severity = severity + self.restart_needed = restart_needed == 1 + + def __str__(self): + return "{} {}".format(self.description_short, self.description_long) + + @property + def is_critical(self): + return self.severity == self.ICS_SPY_ERR_CRITICAL + + class NeoViBus(BusABC): """ The CAN Bus implemented for the python_ics interface @@ -127,10 +156,13 @@ def _process_msg_queue(self, timeout=None): continue self.rx_buffer.append(ics_msg) if errors: - logger.warning("%d errors found" % errors) + logger.warning("%d error(s) found" % errors) for msg in ics.get_error_messages(self.dev): - logger.warning(msg) + error = ICSApiError(*msg) + if error.is_critical: + raise error + logger.warning(error) def _is_filter_match(self, arb_id): """ @@ -219,7 +251,11 @@ def send(self, msg, timeout=None): message.StatusBitField = flags message.StatusBitField2 = 0 message.NetworkID = self.network - ics.transmit_messages(self.dev, message) + + try: + ics.transmit_messages(self.dev, message) + except ics.RuntimeError: + raise ICSApiError(*ics.get_last_api_error(self.dev)) def set_filters(self, can_filters=None): """Apply filtering to all messages received by this Bus. From 3bbc89cbcbba7dd0bdefe1ad9711bd2b3003340f Mon Sep 17 00:00:00 2001 From: oomzay Date: Fri, 9 Feb 2018 13:42:11 +0000 Subject: [PATCH 50/60] VectorBus: Added reset() method. --- can/interfaces/vector/canlib.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/can/interfaces/vector/canlib.py b/can/interfaces/vector/canlib.py index eb18d6e39..19e2c78da 100644 --- a/can/interfaces/vector/canlib.py +++ b/can/interfaces/vector/canlib.py @@ -174,3 +174,9 @@ def shutdown(self): vxlapi.xlDeactivateChannel(self.port_handle, self.mask) vxlapi.xlClosePort(self.port_handle) vxlapi.xlCloseDriver() + + def reset(self): + vxlapi.xlDeactivateChannel(self.port_handle, self.mask) + vxlapi.xlActivateChannel(self.port_handle, self.mask, + vxlapi.XL_BUS_TYPE_CAN, 0) + From 825f52d8b313c51357470c9ac876851cbec8c500 Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sat, 10 Feb 2018 10:03:47 +1100 Subject: [PATCH 51/60] Rename sqlreader for consistency. Closes #229 --- can/__init__.py | 2 +- can/io/__init__.py | 2 +- can/io/player.py | 4 ++-- can/io/sqlite.py | 12 ++++++++---- test/listener_test.py | 2 +- test/logformats_test.py | 2 +- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/can/__init__.py b/can/__init__.py index a15e64999..4a4dac176 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -22,7 +22,7 @@ class CanError(IOError): from can.io import BLFReader, BLFWriter from can.io import CanutilsLogReader, CanutilsLogWriter from can.io import CSVWriter -from can.io import SqliteWriter, SqlReader +from can.io import SqliteWriter, SqliteReader from can.util import set_logging_level diff --git a/can/io/__init__.py b/can/io/__init__.py index 4273abcde..fd2738567 100644 --- a/can/io/__init__.py +++ b/can/io/__init__.py @@ -9,5 +9,5 @@ from .asc import ASCWriter, ASCReader from .blf import BLFReader, BLFWriter from .csv import CSVWriter -from .sqlite import SqlReader, SqliteWriter +from .sqlite import SqliteReader, SqliteWriter from .stdout import Printer diff --git a/can/io/player.py b/can/io/player.py index 15c057bbb..a9f3c07c7 100755 --- a/can/io/player.py +++ b/can/io/player.py @@ -5,7 +5,7 @@ from .asc import ASCReader from .log import CanutilsLogReader from .blf import BLFReader -from .sqlite import SqlReader +from .sqlite import SqliteReader log = logging.getLogger('can.io.player') @@ -35,7 +35,7 @@ def __new__(cls, other, filename): if filename.endswith(".blf"): return BLFReader(filename) if filename.endswith(".db"): - return SqlReader(filename) + return SqliteReader(filename) if filename.endswith(".asc"): return ASCReader(filename) if filename.endswith(".log"): diff --git a/can/io/sqlite.py b/can/io/sqlite.py index 444d2193a..fbb0895a2 100644 --- a/can/io/sqlite.py +++ b/can/io/sqlite.py @@ -19,12 +19,12 @@ buffer = memoryview -class SqlReader: +class SqliteReader: """ Reads recorded CAN messages from a simple SQL database. This class can be iterated over or used to fetch all messages in the - database with :meth:`~SqlReader.read_all`. + database with :meth:`~SqliteReader.read_all`. Calling len() on this object might not run in constant time. """ @@ -32,7 +32,7 @@ class SqlReader: _SELECT_ALL_COMMAND = "SELECT * FROM messages" def __init__(self, filename): - log.debug("Starting SqlReader with %s", filename) + log.debug("Starting SqliteReader with %s", filename) self.conn = sqlite3.connect(filename) self.cursor = self.conn.cursor() @@ -46,7 +46,7 @@ def _create_frame_from_db_tuple(frame_data): def __iter__(self): log.debug("Iterating through messages from sql db") for frame_data in self.cursor.execute(self._SELECT_ALL_COMMAND): - yield SqlReader._create_frame_from_db_tuple(frame_data) + yield SqliteReader._create_frame_from_db_tuple(frame_data) def __len__(self): # this might not run in constant time @@ -63,6 +63,10 @@ def close(self): self.conn.close() +# Backward compatibility +SqlReader = SqliteReader + + class SqliteWriter(BufferedReader): """Logs received CAN data to a simple SQL database. diff --git a/test/listener_test.py b/test/listener_test.py index 40c3ed97c..be795eaf4 100755 --- a/test/listener_test.py +++ b/test/listener_test.py @@ -38,7 +38,7 @@ def testClassesImportable(self): self.assertTrue(hasattr(can, 'CanutilsLogWriter')) self.assertTrue(hasattr(can, 'CanutilsLogReader')) - self.assertTrue(hasattr(can, 'SqlReader')) + self.assertTrue(hasattr(can, 'SqliteReader')) self.assertTrue(hasattr(can, 'SqliteWriter')) self.assertTrue(hasattr(can, 'Printer')) diff --git a/test/logformats_test.py b/test/logformats_test.py index 4bb5fba4f..3cc494adb 100644 --- a/test/logformats_test.py +++ b/test/logformats_test.py @@ -150,7 +150,7 @@ class TestSqlFileFormat(unittest.TestCase): """Tests can.SqliteWriter and can.SqliteReader""" def test_writer_and_reader(self): - _test_writer_and_reader(self, can.SqliteWriter, can.SqlReader, + _test_writer_and_reader(self, can.SqliteWriter, can.SqliteReader, sleep_time=can.SqliteWriter.MAX_TIME_BETWEEN_WRITES, check_comments=False) From d8571e4b6db996bc75065d595e9186c9c72765e0 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Fri, 2 Feb 2018 10:58:15 +0100 Subject: [PATCH 52/60] Added errors='replace' for system with other languages than English --- can/interfaces/pcan/pcan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/can/interfaces/pcan/pcan.py b/can/interfaces/pcan/pcan.py index eb9f6c792..8ff8b162c 100644 --- a/can/interfaces/pcan/pcan.py +++ b/can/interfaces/pcan/pcan.py @@ -130,13 +130,13 @@ def bits(n): if stsReturn[0] != PCAN_ERROR_OK: text = "An error occurred. Error-code's text ({0:X}h) couldn't be retrieved".format(error) else: - text = stsReturn[1].decode('utf-8') + text = stsReturn[1].decode('utf-8', errors='replace') strings.append(text) complete_text = '\n'.join(strings) else: - complete_text = stsReturn[1].decode('utf-8') + complete_text = stsReturn[1].decode('utf-8', errors='replace') return complete_text From 16c61915d52a966b9dc3c890c4df62628ee082d2 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Sun, 11 Feb 2018 15:42:14 +0100 Subject: [PATCH 53/60] Added link in README.rst --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 9c14dc954..97ffb1421 100644 --- a/README.rst +++ b/README.rst @@ -50,4 +50,4 @@ Wherever we interact, we strive to follow the Contributing ------------ -See `doc/development.rst` for getting started. +See `doc/development.rst `__ for getting started. From c474296d12f767f0b295563ec50186b2850165b2 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Mon, 12 Feb 2018 23:39:20 +0100 Subject: [PATCH 54/60] use the same error messages in socketcan_ctypes' send() as in socketcan_native's send() --- can/interfaces/socketcan/socketcan_ctypes.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/can/interfaces/socketcan/socketcan_ctypes.py b/can/interfaces/socketcan/socketcan_ctypes.py index 785c14aac..8d9b2e60d 100644 --- a/can/interfaces/socketcan/socketcan_ctypes.py +++ b/can/interfaces/socketcan/socketcan_ctypes.py @@ -125,12 +125,16 @@ def recv(self, timeout=None): def send(self, msg, timeout=None): frame = _build_can_frame(msg) + if timeout: # Wait for write availability. write will fail below on timeout - select.select([], [self.socket], [], timeout) + _, ready_send_sockets, _ = select.select([], [self.socket], [], timeout) + if not ready_send_sockets: + raise can.CanError("Timeout while sending") + bytes_sent = libc.write(self.socket, ctypes.byref(frame), ctypes.sizeof(frame)) + if bytes_sent == -1: - log.debug("Error sending frame :-/") raise can.CanError("can.socketcan.ctypes failed to transmit") elif bytes_sent == 0: raise can.CanError("Transmit buffer overflow") From 7afacef74aee50ae2c319e2af0a8411bd07a4457 Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Tue, 13 Feb 2018 22:52:35 +0100 Subject: [PATCH 55/60] More test platforms, most notably OSX (#244) * added osx to travis test targets * Improve test failure messages * Make timing tests more forgiving for travis --- .travis.yml | 41 +++++++++++++++++++++++++++++++++++---- README.rst | 4 ++-- can/__init__.py | 2 +- setup.py | 3 +++ test/back2back_test.py | 6 ++++-- test/simplecyclic_test.py | 6 +++--- 6 files changed, 50 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b5989ab4..5ccaa3ff7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,47 @@ language: python -sudo: false + python: + # CPython: - "2.7" - - "pypy" - - "pypy3" + - "3.3" - "3.4" - "3.5" - "3.6" - - "3.7-dev" + - "3.7-dev" # TODO: change to "3.7" once it gets released - "nightly" + # PyPy: + - "pypy" + - "pypy3" + +os: + - linux # Linux is officially supported and we test the library under + # many different Python verions (see "python: ..." above) + +# - osx # OSX + Python is not officially supported by Travis CI as of Feb. 2018 + # nevertheless, "nightly" and some "*-dev" versions seem to work, so we + # include them explicitly below (see "matrix: include: ..." below) + +# - windows # Windows is not supported at all by Travis CI as of Feb. 2018 + +# Linux setup +dist: trusty +sudo: false + +matrix: + # see "os: ..." above + include: + - os: osx + python: "3.6-dev" + - os: osx + python: "3.7-dev" + - os: osx + python: "nightly" + + # allow all nighly builds to fail, since these python versions might be unstable + # we do not allow dev builds to fail, since these builds are stable enough + allow_failures: + - python: "nightly" + install: - travis_retry pip install . - travis_retry pip install -r requirements.txt diff --git a/README.rst b/README.rst index 97ffb1421..8002997eb 100644 --- a/README.rst +++ b/README.rst @@ -12,7 +12,7 @@ python-can :alt: Documentation Status .. |build| image:: https://travis-ci.org/hardbyte/python-can.svg?branch=develop - :target: https://travis-ci.org/hardbyte/python-can + :target: https://travis-ci.org/hardbyte/python-can/branches :alt: CI Server for develop branch @@ -26,7 +26,7 @@ Python developers; providing `common abstractions to different hardware devices`, and a suite of utilities for sending and receiving messages on a can bus. -The library supports Python 2.7, Python 3.3+ and runs on Mac, Linux and Windows. +The library supports Python 2.7, Python 3.3+ as well as PyPy and runs on Mac, Linux and Windows. You can find more information in the documentation, online at `python-can.readthedocs.org `__. diff --git a/can/__init__.py b/can/__init__.py index 4a4dac176..70d3b2531 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -5,7 +5,7 @@ import logging -__version__ = "2.0.0" +__version__ = "2.1.0.rc2" log = logging.getLogger('can') diff --git a/setup.py b/setup.py index 8813466ed..98bf4a871 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,9 @@ +# -*- coding: utf-8 -*- + """ python-can requires the setuptools package to be installed. """ + import re import logging from setuptools import setup, find_packages diff --git a/test/back2back_test.py b/test/back2back_test.py index c20aac5e2..ef6e884f8 100644 --- a/test/back2back_test.py +++ b/test/back2back_test.py @@ -70,11 +70,13 @@ def test_no_message(self): def test_timestamp(self): self.bus2.send(can.Message()) recv_msg1 = self.bus1.recv(TIMEOUT) - time.sleep(1) + time.sleep(5) self.bus2.send(can.Message()) recv_msg2 = self.bus1.recv(TIMEOUT) delta_time = recv_msg2.timestamp - recv_msg1.timestamp - self.assertTrue(0.95 < delta_time < 1.05) + self.assertTrue(4.8 < delta_time < 5.2, + 'Time difference should have been 5s +/- 200ms.' + 'But measured {}'.format(delta_time)) def test_standard_message(self): msg = can.Message(extended_id=False, diff --git a/test/simplecyclic_test.py b/test/simplecyclic_test.py index e3aed2126..822827c42 100644 --- a/test/simplecyclic_test.py +++ b/test/simplecyclic_test.py @@ -12,11 +12,11 @@ def test_cycle_time(self): task = bus.send_periodic(msg, 0.01, 1) self.assertIsInstance(task, can.broadcastmanager.CyclicSendTaskABC) - sleep(1.5) + sleep(5) size = bus2.queue.qsize() - print(size) # About 100 messages should have been transmitted - self.assertTrue(90 < size < 110) + self.assertTrue(90 < size < 110, + '100 +/- 10 messages should have been transmitted. But queue contained {}'.format(size)) last_msg = bus2.recv() self.assertEqual(last_msg, msg) From 269be72934e8f0cdcf5a3f4b19ac793fd005527c Mon Sep 17 00:00:00 2001 From: Felix Divo Date: Fri, 16 Feb 2018 18:06:00 +0100 Subject: [PATCH 56/60] fixes #243 : skip timing/performance sensitive tests if running on Travis CI --- test/back2back_test.py | 3 +++ test/simplecyclic_test.py | 11 +++++++---- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/test/back2back_test.py b/test/back2back_test.py index ef6e884f8..202a5365e 100644 --- a/test/back2back_test.py +++ b/test/back2back_test.py @@ -1,8 +1,10 @@ +import os import unittest import time import can +IS_TRAVIS = os.environ.get('TRAVIS', 'default') == 'true' BITRATE = 500000 TIMEOUT = 0.1 @@ -67,6 +69,7 @@ def _send_and_receive(self, msg): def test_no_message(self): self.assertIsNone(self.bus1.recv(0.1)) + @unittest.skipIf(IS_TRAVIS, "skip on Travis CI") def test_timestamp(self): self.bus2.send(can.Message()) recv_msg1 = self.bus1.recv(TIMEOUT) diff --git a/test/simplecyclic_test.py b/test/simplecyclic_test.py index 822827c42..980fe1eee 100644 --- a/test/simplecyclic_test.py +++ b/test/simplecyclic_test.py @@ -1,15 +1,19 @@ +import os from time import sleep import unittest + import can +IS_TRAVIS = os.environ.get('TRAVIS', 'default') == 'true' class SimpleCyclicSendTaskTest(unittest.TestCase): + @unittest.skipIf(IS_TRAVIS, "skip on Travis CI") def test_cycle_time(self): msg = can.Message(extended_id=False, arbitration_id=0x100, data=[0,1,2,3,4,5,6,7]) - bus = can.interface.Bus(bustype='virtual') + bus1 = can.interface.Bus(bustype='virtual') bus2 = can.interface.Bus(bustype='virtual') - task = bus.send_periodic(msg, 0.01, 1) + task = bus1.send_periodic(msg, 0.01, 1) self.assertIsInstance(task, can.broadcastmanager.CyclicSendTaskABC) sleep(5) @@ -20,9 +24,8 @@ def test_cycle_time(self): last_msg = bus2.recv() self.assertEqual(last_msg, msg) - bus.shutdown() + bus1.shutdown() bus2.shutdown() - if __name__ == '__main__': unittest.main() From 10aef5032ce78b699286af5c3796fc1b05018c4b Mon Sep 17 00:00:00 2001 From: pierreluctg Date: Fri, 16 Feb 2018 20:16:31 -0500 Subject: [PATCH 57/60] Adding neovi device serial base 36 decoding when needed (#250) --- can/interfaces/ics_neovi/neovi_bus.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/can/interfaces/ics_neovi/neovi_bus.py b/can/interfaces/ics_neovi/neovi_bus.py index 6402d59f8..5e889091c 100644 --- a/can/interfaces/ics_neovi/neovi_bus.py +++ b/can/interfaces/ics_neovi/neovi_bus.py @@ -98,7 +98,7 @@ def __init__(self, channel=None, can_filters=None, **config): self.channel_info = '%s %s CH:%s' % ( self.dev.Name, - self.dev.SerialNumber, + self.get_serial_number(self.dev), channel ) logger.info("Using device: {}".format(self.channel_info)) @@ -117,6 +117,23 @@ def __init__(self, channel=None, can_filters=None, **config): ics.NEOVI6_VCAN_TIMESTAMP_1, ics.NEOVI6_VCAN_TIMESTAMP_2 ) + @staticmethod + def get_serial_number(device): + """Decode (if needed) and return the ICS device serial string + + :param device: ics device + :return: ics device serial string + :rtype: str + """ + def to_base36(n, alphabet="0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"): + return (to_base36(n // 36) + alphabet[n % 36]).lstrip("0") \ + if n > 0 else "0" + + a0000 = 604661760 + if device.SerialNumber >= a0000: + return to_base36(device.SerialNumber) + return str(device.SerialNumber) + def shutdown(self): super(NeoViBus, self).shutdown() self.opened = False @@ -129,7 +146,7 @@ def _open_device(self, type_filter=None, serial=None): devices = ics.find_devices() for device in devices: - if serial is None or str(device.SerialNumber) == str(serial): + if serial is None or self.get_serial_number(device) == str(serial): dev = device break else: From 791307c9d8b733cc3357533fb74cf54a0dec6e2c Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sat, 10 Feb 2018 13:24:59 +1100 Subject: [PATCH 58/60] Add changelog for v2.1.0 --- CHANGELOG.txt | 34 ++++++++++++++++++++++++++++++++++ can/__init__.py | 2 +- doc/development.rst | 2 +- 3 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 CHANGELOG.txt diff --git a/CHANGELOG.txt b/CHANGELOG.txt new file mode 100644 index 000000000..07b75975b --- /dev/null +++ b/CHANGELOG.txt @@ -0,0 +1,34 @@ +Version 2.1.0 (2018-02-10) +===== + + +* Support for out of tree can interfaces with pluggy. +* Neovi interface now uses Intrepid Control Systems's own interface library. +* Improvements and new documentation for SQL reader/writer + + +Version 2.0.0 (2018-01-05 +===== + +After an extended baking period we have finally tagged version 2.0.0! + +Quite a few major Changes from v1.x: + +* New interfaces: + * Vector + * NI-CAN + * isCAN + * neoVI +* Simplified periodic send API with initial support for SocketCAN +* Protocols module including J1939 support removed +* Logger script moved to module `can.logger` +* New `can.player` script to replay log files +* BLF, ASC log file support added in new `can.io` module + +You can install from [PyPi](https://pypi.python.org/pypi/python-can/2.0.0) with pip: + +``` +pip install python-can==2.0.0 +``` + +The documentation for v2.0.0 is available at http://python-can.readthedocs.io/en/2.0.0/ diff --git a/can/__init__.py b/can/__init__.py index 70d3b2531..71bc0f442 100644 --- a/can/__init__.py +++ b/can/__init__.py @@ -5,7 +5,7 @@ import logging -__version__ = "2.1.0.rc2" +__version__ = "2.1.0" log = logging.getLogger('can') diff --git a/doc/development.rst b/doc/development.rst index 6fb0de2f4..f7e09b671 100644 --- a/doc/development.rst +++ b/doc/development.rst @@ -38,7 +38,7 @@ Creating a Release - Upload with twine ``twine upload dist/python-can-X.Y.Z*`` - In a new virtual env check that the package can be installed with pip: ``pip install python-can==X.Y.Z`` - Create a new tag in the repository. -- Check the release on PyPi and github. +- Check the release on PyPi, readthedocs and github. Code Structure From cd53ec42c8c69d725ca9ce9ccc22f0ba94c09a3d Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sat, 10 Feb 2018 13:33:45 +1100 Subject: [PATCH 59/60] Update changelog for v2.1.0 --- CHANGELOG.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 07b75975b..e81cedd16 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,11 +1,13 @@ -Version 2.1.0 (2018-02-10) +Version 2.1.0 (2018-02-17) ===== - * Support for out of tree can interfaces with pluggy. +* Initial support for CAN-FD for socketcan_native and kvaser interfaces. * Neovi interface now uses Intrepid Control Systems's own interface library. -* Improvements and new documentation for SQL reader/writer - +* Improvements and new documentation for SQL reader/writer. +* Fix bug in neovi serial number decoding. +* Add testing on OSX to TravisCI +* Fix non english decoding error on pcan Version 2.0.0 (2018-01-05 ===== From 9ca495263d9815ab3c3d0edbee15744cc7ad60ca Mon Sep 17 00:00:00 2001 From: Brian Thorne Date: Sun, 18 Feb 2018 17:03:53 +1100 Subject: [PATCH 60/60] Add misc fixes note to changelog --- CHANGELOG.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index e81cedd16..06b6ef4ee 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -8,6 +8,8 @@ Version 2.1.0 (2018-02-17) * Fix bug in neovi serial number decoding. * Add testing on OSX to TravisCI * Fix non english decoding error on pcan +* Other misc improvements and bug fixes + Version 2.0.0 (2018-01-05 =====