From f4d4339230296c19528c3497aac636943b8b5051 Mon Sep 17 00:00:00 2001 From: Fabian Frisch <11352551+fabm3n@users.noreply.github.com> Date: Sun, 23 Oct 2022 21:25:41 +0200 Subject: [PATCH 1/4] fix sqlite migration --- .gitignore | 1 + README.md | 2 +- homeassistant2influxdb.py | 15 +++++++++------ requirements.txt | 10 +++++----- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 7b3a989..158008c 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ home-assistant-core .python-version .idea/ venv/ +.vscode/ \ No newline at end of file diff --git a/README.md b/README.md index d20c949..c84b4e3 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Setup: 1. `sudo apt install python3 python3.7-dev` 2. `git clone migrate2influxdb` 3. `cd migrate2influxdb` -4. `git clone git@github.com:home-assistant/core.git home-assistant-core` +4. `git clone https://github.com/home-assistant/core.git home-assistant-core` 5. `python3 -m venv .venv` Dependency installation: diff --git a/homeassistant2influxdb.py b/homeassistant2influxdb.py index b790e8f..5ab9806 100755 --- a/homeassistant2influxdb.py +++ b/homeassistant2influxdb.py @@ -10,6 +10,7 @@ from tqdm import tqdm import voluptuous as vol import yaml +from datetime import datetime sys.path.append("home-assistant-core") @@ -211,7 +212,7 @@ def main(): event = Event( _event_type, data={"new_state": state}, - time_fired=_time_fired + time_fired=datetime.strptime(_time_fired, "%Y-%m-%d %H:%M:%S.%f") ) except InvalidEntityFormatError: pass @@ -253,7 +254,9 @@ def main(): # Clean up by closing influx connection, and removing temporary table influx.close() if args.table == 'both': - remove_tmp_table() + cursor = connection.cursor() + remove_tmp_table(cursor) + cursor.close() # print statistics - ideally you have one friendly name per entity_id # you can use the output to see where the same sensor has had different @@ -289,7 +292,7 @@ def formulate_sql_query(table: str, arg_tables: str): if table == "states": # Using two different SQL queries in a Union to support data made with older HA db schema: # https://github.com/home-assistant/core/pull/71165 - sql_query = """select SQL_NO_CACHE states.entity_id, + sql_query = """select states.entity_id, states.state, states.attributes, events.event_type as event_type, @@ -314,7 +317,7 @@ def formulate_sql_query(table: str, arg_tables: str): else: inset_query = '' sql_query = f""" - SELECT SQL_NO_CACHE statistics_meta.statistic_id, + SELECT statistics_meta.statistic_id, statistics.mean, statistics.min, statistics.max, @@ -342,7 +345,7 @@ def formulate_tmp_table_sql(): TODO Not perfect solution, some entities have multiple attributes that change over time. TODO Here we select the most recent """ - return """CREATE TEMPORARY TABLE IF NOT EXISTS state_tmp + return """CREATE TEMPORARY TABLE IF NOT EXISTS state_tmp AS SELECT max(states.attributes_id) as attributes_id, states.entity_id FROM states WHERE states.attributes_id IS NOT NULL @@ -351,7 +354,7 @@ def formulate_tmp_table_sql(): def remove_tmp_table(cursor): - cursor.execute("""DROP TEMPORARY TABLE state_tmp;""") + cursor.execute("""DROP TABLE IF EXISTS state_tmp;""") if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 3ec7dd6..0b5ff54 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ influxdb influxdb_client -mysqlclient~=2.1.1 -pyyaml~=6.0 +mysqlclient +pyyaml setuptools -tqdm~=4.64.0 +tqdm wheel -voluptuous~=0.13.1 -homeassistant~=2022.6.7 \ No newline at end of file +voluptuous +homeassistant \ No newline at end of file From 1b0ee1174e4f334ad842d8140e81ab80af8c5c66 Mon Sep 17 00:00:00 2001 From: Fabian <11352551+fabm3n@users.noreply.github.com> Date: Thu, 2 Feb 2023 14:11:50 +0000 Subject: [PATCH 2/4] fix clone --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 465db41..56b54ac 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Setup: 1. `sudo apt install python3 python3.7-dev` 2. `git clone migrate2influxdb` 3. `cd migrate2influxdb` -4. `git clone --depth=1 https://github.com:home-assistant/core.git home-assistant-core` +4. `git clone --depth=1 https://github.com/home-assistant/core.git home-assistant-core` 5. `python3 -m venv .venv` Dependency installation: From 10b52c15f4a2c0befc64cffe033ad75e9a075b43 Mon Sep 17 00:00:00 2001 From: Fabian <11352551+fabm3n@users.noreply.github.com> Date: Fri, 3 Feb 2023 12:29:06 +0000 Subject: [PATCH 3/4] fix datetime conversion for mariadb --- homeassistant2influxdb.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/homeassistant2influxdb.py b/homeassistant2influxdb.py index 5ab9806..1ca1c19 100755 --- a/homeassistant2influxdb.py +++ b/homeassistant2influxdb.py @@ -209,11 +209,18 @@ def main(): entity_id=_entity_id, state=_state, attributes=_attributes) - event = Event( - _event_type, - data={"new_state": state}, - time_fired=datetime.strptime(_time_fired, "%Y-%m-%d %H:%M:%S.%f") - ) + if args.type == "MySQL" or args.type == "MariaDB": + event = Event( + _event_type, + data={"new_state": state}, + time_fired=_time_fired + ) + else: + event = Event( + _event_type, + data={"new_state": state}, + time_fired=datetime.strptime(_time_fired, "%Y-%m-%d %H:%M:%S.%f") + ) except InvalidEntityFormatError: pass else: From a146d383218a11bdac003606b74ca83f0097a2ce Mon Sep 17 00:00:00 2001 From: Fabian <11352551+fabm3n@users.noreply.github.com> Date: Fri, 3 Feb 2023 15:12:08 +0000 Subject: [PATCH 4/4] convert json only when not None --- homeassistant2influxdb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/homeassistant2influxdb.py b/homeassistant2influxdb.py index 1ca1c19..d1ee246 100755 --- a/homeassistant2influxdb.py +++ b/homeassistant2influxdb.py @@ -186,7 +186,10 @@ def main(): _entity_id = rename_entity_id(row[0]) _state = row[1] _attributes_raw = row[2] - _attributes = rename_friendly_name(json.loads(_attributes_raw)) + if _attributes_raw is None: + _attributes = None + else: + _attributes = rename_friendly_name(json.loads(_attributes_raw)) _event_type = row[3] _time_fired = row[4] elif table == "statistics":