From 6e00d192417d407d0d9af7f98a6524c10e0ac04e Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 24 Dec 2022 18:41:52 +0000 Subject: [PATCH 01/31] Support MQTT --- README.md | 133 ++---------------- ...a_switch_to_topic_from_event_descriptor.py | 35 +++++ rogerthat/config/config.py | 16 ++- rogerthat/config/loader.py | 6 + .../config/samples/gateway_mqtt.sample.yml | 14 ++ .../config/samples/main_config.sample.yml | 6 +- .../config/samples/tradingview.sample.yml | 7 +- .../config/samples/web_server.sample.yml | 12 +- rogerthat/config/utils.py | 32 ++--- rogerthat/db/models/tradingview_event.py | 81 ++++++----- rogerthat/models/web_request.py | 12 +- rogerthat/models/wss_request.py | 70 --------- rogerthat/mqtt/messages.py | 113 +++++++++++++++ rogerthat/mqtt/mqtt.py | 74 ++++++++++ rogerthat/queues/mqtt_queue.py | 61 ++++++++ rogerthat/queues/ws_queue.py | 67 --------- .../route_handlers/handlers/hummingbot.py | 44 ------ .../route_handlers/route_handlers_ctrl.py | 9 +- rogerthat/server/routes.py | 25 +--- rogerthat/utils/splash.py | 2 +- scripts/setup.py | 12 -- support/environment.yml | 2 + tests/test_websocket.py | 30 ---- tests/test_websocket_filtered.py | 30 ---- 24 files changed, 397 insertions(+), 496 deletions(-) create mode 100644 alembic/versions/b492ff9dde0a_switch_to_topic_from_event_descriptor.py create mode 100644 rogerthat/config/samples/gateway_mqtt.sample.yml delete mode 100644 rogerthat/models/wss_request.py create mode 100644 rogerthat/mqtt/messages.py create mode 100644 rogerthat/mqtt/mqtt.py create mode 100644 rogerthat/queues/mqtt_queue.py delete mode 100644 rogerthat/queues/ws_queue.py delete mode 100644 rogerthat/route_handlers/handlers/hummingbot.py delete mode 100755 tests/test_websocket.py delete mode 100644 tests/test_websocket_filtered.py diff --git a/README.md b/README.md index 41f46f0..0c25691 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ This file is auto-generated by a github hook, modify docs/README.template.md instead. --> # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! -## This is an outdated version and no longer supported! See the `mqtt` branch for the latest version. +## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge # RogerThat @@ -10,7 +10,7 @@ This file is auto-generated by a github hook, modify docs/README.template.md ins **TradingView** is a widely used market market tracker that allows users to create, share and use custom strategies but is quite limited in how it can be "plugged in" to other services. It does however allow the creation of "**Alerts**" which can send data to a webhook (public URL). -**RogerThat** facilitates the collection and forwarding of these **TradingView Alerts** to **Hummingbot** via the **Remote Command Executor** module, which listens to **RogerThat** via a websocket for received commands and updates. +**RogerThat** facilitates the collection and forwarding of these **TradingView Alerts** to **Hummingbot** via the **MQTT Bridge** module, which listens to **RogerThat** via subscribed MQTT topics. Whilst it's purpose is to bridge **TradingView** and **Hummingbot**, it can work as a gateway / bridge between any service that sends data via Webhooks (to a public URL) and serve / route them to *multiple* **Hummingbot** instances. @@ -306,12 +306,9 @@ Where `` is your domain name or public IP address and ### JSON Data for TradingView alerts. -Alerts must be formatted as JSON, the only required parameter is `name`. -*(this parameter key name is configurable in `configs/tradingview.yml`)* +Alerts must be formatted as JSON, the only required parameter is `topic`. -The `name` key or one of the `tradingview_descriptor_fields` must be present in the JSON data to be accepted, but the value can be null or empty. This is transmitted to **Hummingbot** as `event_descriptor` - -Using an empty value for `name` ignores **Hummingbot**'s *Remote Command Executor* routing names. +The `topic` key must be present in the JSON data to be accepted.
Example alert data: @@ -356,92 +353,27 @@ ___ ## Hummingbot Connection -You can connect the **Hummingbot** _Remote Command Executor_ to the **RogerThat** websocket with this URL: -```html -ws://localhost:10073/wss -``` +Connect the **Hummingbot** _MQTT Bridge_ and **RogerThat** to the same MQTT Broker. ### Example Config (NEW)
-Example Config (NEW) ... +Example Config ... -Use something like the following config to connect **RogerThat** to **Hummingbot** via the **Remote Command Executor**. +Use something like the following config to connect **RogerThat** to **Hummingbot** via the **MQTT Bridge**. This config is found inside your main hummingbot folder then `conf\conf_client.yml` ```yaml # Remote commands -remote_command_executor_mode: - remote_command_executor_api_key: a9ba4b61-6f6d-41cf-85c3-7cfdfcbea0f3 - remote_command_executor_ws_url: ws://localhost:10073/wss - # Specify a routing name (for use with multiple Hummingbot instances) - remote_command_executor_routing_name: - # Recommended to keep this on so no events are missed in the case of a network drop out. - remote_command_executor_ignore_first_event: true - # Whether to disable console command processing for remote command events. - # Best to disable this if using in custom scripts or strategies - remote_command_executor_disable_console_commands: false - # You can specify how to translate received commands to Hummingbot commands here - # eg. - # remote_commands_translate_commands: - # long: start - # short: stop - remote_command_executor_translate_commands: - long: start - short: stop +mqtt_bridge: + mqtt_host: localhost + mqtt_port: 1883 + mqtt_autostart: true ```
-### Example Config (OLD) - -
-Example Config (OLD) ... - -Use something like the following config to connect **RogerThat** to **Hummingbot** via the **Remote Command Executor**. - -This config is found inside your main hummingbot folder then `conf\conf_global.yml` - -```yaml -# Remote commands -remote_commands_enabled: true -remote_commands_api_key: a9ba4b61-6f6d-41cf-85c3-7cfdfcbea0f3 -remote_commands_ws_url: ws://localhost:10073/wss -# Specify a routing name (for use with multiple Hummingbot instances) -remote_commands_routing_name: -# Recommended to keep this on so no events are missed in the case of a network drop out. -remote_commands_ignore_first_event: true -# Whether to disable console command processing for remote command events. -# Best to disable this if using in custom scripts or strategies -remote_commands_disable_console_commands: false -# You can specify how to translate received commands to Hummingbot commands here -# eg. -# remote_commands_translate_commands: -# long: start -# short: stop -remote_commands_translate_commands: - long: start - short: stop -``` - -
- - -### Filtering events - -
-Expand ... - -To filter events received based on the `event_descriptor`, change your websockets URL to: -```html -ws://localhost:10073/wss/ -``` - -Where `event-descriptor` matches the `event_descriptor` or *name* value of the events you wish to receive. - -
- ### Command Shortcuts @@ -459,48 +391,7 @@ ___
Test Connection ... -You can enable and disable websockets authentication in the config with these commands. -(You must disable websockets authentication for the in-browser test to work.) - -
-Linux/Mac - -```bash -scripts/setup_config.sh --enable-websocket-auth -scripts/setup_config.sh --disable-websocket-auth -``` -
-
-Windows - -```bat -scripts\setup_config.bat --enable-websocket-auth -scripts\setup_config.bat --disable-websocket-auth -``` -
- -Test the websocket feed in your browser with this js code: - -```javascript -var ws = new WebSocket('ws://localhost:10073/wss'); -ws.onmessage = function (event) { - console.log(event.data); -}; -``` - -Or run the python test listener (requires source installation steps but can authenticate): - -```bash -python tests/test_websocket.py -``` - -Or test the REST url here: - -```html -http://localhost:10073/api/hbot/?api_key= -``` - -Where `` is the generated api key found in `web_server.yml` under `api_allowed_keys_hbot`. +To test basic connection, use any MQTT client and connect to the same broker as RogerThat, then subscribe to the `rogerthat/#` topic.
diff --git a/alembic/versions/b492ff9dde0a_switch_to_topic_from_event_descriptor.py b/alembic/versions/b492ff9dde0a_switch_to_topic_from_event_descriptor.py new file mode 100644 index 0000000..01f809c --- /dev/null +++ b/alembic/versions/b492ff9dde0a_switch_to_topic_from_event_descriptor.py @@ -0,0 +1,35 @@ +"""switch to topic from event_descriptor + +Revision ID: b492ff9dde0a +Revises: 49158b1dc3fa +Create Date: 2022-12-24 17:34:05.168925+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'b492ff9dde0a' +down_revision = '49158b1dc3fa' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tradingview_events', sa.Column('topic', sa.String(length=220), nullable=True)) + op.drop_index('ix_tradingview_events_event_descriptor', table_name='tradingview_events') + op.create_index(op.f('ix_tradingview_events_topic'), 'tradingview_events', ['topic'], unique=False) + op.drop_column('tradingview_events', 'event_descriptor') + op.execute("delete from tradingview_events") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tradingview_events', sa.Column('event_descriptor', sa.VARCHAR(length=100), autoincrement=False, nullable=True)) + op.drop_index(op.f('ix_tradingview_events_topic'), table_name='tradingview_events') + op.create_index('ix_tradingview_events_event_descriptor', 'tradingview_events', ['event_descriptor'], unique=False) + op.drop_column('tradingview_events', 'topic') + # ### end Alembic commands ### diff --git a/rogerthat/config/config.py b/rogerthat/config/config.py index e3d8f30..affd4c4 100644 --- a/rogerthat/config/config.py +++ b/rogerthat/config/config.py @@ -7,6 +7,7 @@ _loaded_configs = config_loader() _app_config = _loaded_configs.app_config _db_config = _loaded_configs.db_config +_mqtt_config = _loaded_configs.mqtt_config _tv_config = _loaded_configs.tv_config _web_config = _loaded_configs.web_config @@ -16,13 +17,11 @@ class ConfigSetup(no_setters): _project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) _app_name = _app_config['app_name'] _debug_mode = _app_config['debug_mode'] - _rebroadcast_on_ws_connect = _app_config.get('rebroadcast_on_ws_connect', True) _include_extra_order_fields = _app_config.get('include_extra_order_fields', False) # Web _server_host = _web_config['server_host'] _web_root = _web_config['web_root'] - _wss_root = _web_config['wss_root'] _quart_secret_key = _web_config['quart_secret_key'] _quart_cookie_domain_debug = _web_config['quart_cookie_domain_debug'] _quart_auth_pep = _web_config['quart_auth_pep'] @@ -30,6 +29,15 @@ class ConfigSetup(no_setters): _quart_server_port = _web_config['quart_server_port'] _protect_with_cloudflare_firewall_rules = _web_config.get('protect_with_cloudflare_firewall_rules', False) + # MQTT + _mqtt_enable = _mqtt_config['mqtt_enable'] + _mqtt_instance_name = _mqtt_config['mqtt_instance_name'] + _mqtt_host = _mqtt_config['mqtt_host'] + _mqtt_port = _mqtt_config['mqtt_port'] + _mqtt_username = _mqtt_config['mqtt_username'] + _mqtt_password = _mqtt_config['mqtt_password'] + _mqtt_ssl = _mqtt_config['mqtt_ssl'] + # Database _database_protocol = _db_config.get("database_protocol", "postgresql+asyncpg") _database_host = _db_config.get("database_host", "localhost") @@ -41,11 +49,9 @@ class ConfigSetup(no_setters): # Security _accepted_user_agents_tv = _web_config['accepted_user_agents_tv'] _api_allowed_keys_tv = _web_config['api_allowed_keys_tv'] - _api_allowed_keys_hbot = _web_config['api_allowed_keys_hbot'] - _disable_websocket_authentication = _web_config.get('disable_websocket_authentication', False) # Trading View - _tradingview_descriptor_fields = _tv_config['tradingview_descriptor_fields'] + _tradingview_translations = _tv_config['tradingview_translations'] Config = ConfigSetup() diff --git a/rogerthat/config/loader.py b/rogerthat/config/loader.py index c50a775..20335fd 100644 --- a/rogerthat/config/loader.py +++ b/rogerthat/config/loader.py @@ -7,6 +7,7 @@ class config_loader: def __init__(self, *args, **kwargs): self._app_config = None self._db_config = None + self._mqtt_config = None self._tv_config = None self._web_config = None self.setup_configs() @@ -19,6 +20,10 @@ def app_config(self): def db_config(self): return self._db_config + @property + def mqtt_config(self): + return self._mqtt_config + @property def tv_config(self): return self._tv_config @@ -30,6 +35,7 @@ def web_config(self): def load_configs(self): self._app_config = config_utils.load_config_app() self._db_config = config_utils.load_config_db() + self._mqtt_config = config_utils.load_config_mqtt() self._tv_config = config_utils.load_config_tv() self._web_config = config_utils.load_config_web() diff --git a/rogerthat/config/samples/gateway_mqtt.sample.yml b/rogerthat/config/samples/gateway_mqtt.sample.yml new file mode 100644 index 0000000..aca99a0 --- /dev/null +++ b/rogerthat/config/samples/gateway_mqtt.sample.yml @@ -0,0 +1,14 @@ +######################################### +### Gateway - MQTT config ### +######################################### +template_version: 0.1 + +mqtt_enable: true + +mqtt_instance_name: some-uid + +mqtt_host: localhost +mqtt_port: 1883 +mqtt_username: user +mqtt_password: password +mqtt_ssl: false diff --git a/rogerthat/config/samples/main_config.sample.yml b/rogerthat/config/samples/main_config.sample.yml index 909b11b..7670994 100644 --- a/rogerthat/config/samples/main_config.sample.yml +++ b/rogerthat/config/samples/main_config.sample.yml @@ -1,15 +1,11 @@ ######################################### ### Main App config ### ######################################### -template_version: 0.1 +template_version: 0.2 # Main server app name app_name: rogerthat -# Rebroadcast last received event on websocket first connection. -# Recommended to keep this on in case events are missed during a network drop out. -rebroadcast_on_ws_connect: true - # ADVANCED! # Setting this option to `true` will include extra order fields in sent data. # These fields won't do anything without your intervention, they require scripting of Hummingbot. diff --git a/rogerthat/config/samples/tradingview.sample.yml b/rogerthat/config/samples/tradingview.sample.yml index d3510cb..661c7e9 100644 --- a/rogerthat/config/samples/tradingview.sample.yml +++ b/rogerthat/config/samples/tradingview.sample.yml @@ -1,11 +1,10 @@ ######################################### ### TradingView config ### ######################################### -template_version: 0.1 +template_version: 0.2 # A list of key names to use for the event descriptor on received events. # Only one will be used. -# Defaults to `event_descriptor` if none set. # Event data must contain a descriptor key to be accepted. -tradingview_descriptor_fields: -- name +tradingview_translations: + name: translated_name diff --git a/rogerthat/config/samples/web_server.sample.yml b/rogerthat/config/samples/web_server.sample.yml index 944903c..cf2fbf6 100644 --- a/rogerthat/config/samples/web_server.sample.yml +++ b/rogerthat/config/samples/web_server.sample.yml @@ -1,7 +1,7 @@ ######################################### ### Web Server config ### ######################################### -template_version: 0.2 +template_version: 0.3 # Server HOSTNAME to listen on publicly # (Just the HOSTNAME, not the full URL) @@ -9,21 +9,11 @@ server_host: localhost # Webroot for REST URLs web_root: api -# Webroot for Websocket URLs -wss_root: wss - -# Allowed API keys for the Hummingbot REST route and Websocket route -api_allowed_keys_hbot: -- apikey1 # Allowed API keys for the TradingView route api_allowed_keys_tv: - apikey1 -# Whether to disable authentication on the Websocket route. -# (Only recommended for testing.) -disable_websocket_authentication: false - # Accepted user agents on the TradingView router accepted_user_agents_tv: - go-http-client/1.1 diff --git a/rogerthat/config/utils.py b/rogerthat/config/utils.py index 2d0d4c5..07f4586 100644 --- a/rogerthat/config/utils.py +++ b/rogerthat/config/utils.py @@ -20,9 +20,10 @@ class config_utils: # Config file names _conf_file_db = "database" _conf_file_main = "main_config" + _conf_file_mqtt = "gateway_mqtt" _conf_file_tv = "tradingview" _conf_file_web = "web_server" - _conf_file_list = [_conf_file_db, _conf_file_main, _conf_file_tv, _conf_file_web] + _conf_file_list = [_conf_file_db, _conf_file_main, _conf_file_mqtt, _conf_file_tv, _conf_file_web] _auto_gen_str = "# This file is auto-generated, changes will be overwritten by .yml values\n\n" @@ -51,6 +52,12 @@ def generate_api_key(cls, existing_keys): new_api_key = api_key return new_api_key + @classmethod + def generate_mqtt_instance_name(cls): + config = cls.load_config(cls._conf_file_mqtt) + config["mqtt_instance_name"] = str(uuid.uuid4()) + cls.save_config(config, cls._conf_file_mqtt) + @classmethod def generate_quart_secrets(cls): config = cls.load_config(cls._conf_file_web) @@ -68,15 +75,6 @@ def save_new_api_key_tv(cls, clear=False): config["api_allowed_keys_tv"] = yml_add_to_list(config["api_allowed_keys_tv"], newkey) cls.save_config(config, cls._conf_file_web) - @classmethod - def save_new_api_key_hbot(cls, clear=False): - config = cls.load_config(cls._conf_file_web) - newkey = cls.generate_api_key(config["api_allowed_keys_hbot"]) - if clear: - config["api_allowed_keys_hbot"] = yml_clear_list(config["api_allowed_keys_hbot"]) - config["api_allowed_keys_hbot"] = yml_add_to_list(config["api_allowed_keys_hbot"], newkey) - cls.save_config(config, cls._conf_file_web) - @classmethod def update_conf_from_template(cls, conf_file): sample_config = cls.load_config(conf_file, sample_mode=True) @@ -126,12 +124,6 @@ def toggle_iptables(cls, enable=False): cls.save_config(config, cls._conf_file_web) cls.generate_env_nginx() - @classmethod - def toggle_websocket_auth(cls, disable): - config = cls.load_config(cls._conf_file_web) - config["disable_websocket_authentication"] = bool(disable) - cls.save_config(config, cls._conf_file_web) - @classmethod def generate_env_postgres(cls, safe=False): pg_env_path = os.path.join(cls._config_dir, "env_postgres.env") @@ -176,10 +168,12 @@ def copy_fresh_templates(cls, safe=False): continue os.remove(new_conf) shutil.copy(templ_conf, new_conf) + if "gateway_mqtt" in new_conf: + cls.generate_mqtt_instance_name() if not safe or not configs_exist: cls.save_new_api_key_tv(clear=True) - cls.save_new_api_key_hbot(clear=True) cls.generate_quart_secrets() + cls.generate_mqtt_instance_name() cls.generate_env_postgres() cls.generate_env_nginx() @@ -211,6 +205,10 @@ def load_config_app(cls): def load_config_db(cls): return cls.load_config(cls._conf_file_db) + @classmethod + def load_config_mqtt(cls): + return cls.load_config(cls._conf_file_mqtt) + @classmethod def load_config_tv(cls): return cls.load_config(cls._conf_file_tv) diff --git a/rogerthat/db/models/tradingview_event.py b/rogerthat/db/models/tradingview_event.py index b091192..994d735 100644 --- a/rogerthat/db/models/tradingview_event.py +++ b/rogerthat/db/models/tradingview_event.py @@ -1,12 +1,13 @@ from decimal import Decimal as Dec import time -import ujson +from commlib.msg import PubSubMessage from sqlalchemy import ( Column, BigInteger, Numeric, String, ) +from typing import Optional from rogerthat.config.config import Config from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select @@ -15,7 +16,8 @@ base_model, db_model_base, ) -from rogerthat.queues.ws_queue import ws_queue +from rogerthat.queues.mqtt_queue import mqtt_queue +# from rogerthat.queues.ws_queue import ws_queue from rogerthat.utils.parsing_numbers import ( decimal_or_none, decimal_or_zero, @@ -27,13 +29,39 @@ logger = AsyncioLogger.get_logger_main(__name__) +class pydantic_tradingview_event_base(PubSubMessage): + id: int + timestamp_received: int + timestamp_event: Optional[int] = None + topic: str + command: Optional[str] = None + exchange: Optional[str] = None + symbol: Optional[str] = None + interval: Optional[int] = None + price: Optional[Dec] = None + volume: Optional[Dec] = None + inventory: Optional[Dec] = None + + class Config: + orm_mode = True + + +class pydantic_tradingview_event_with_order(pydantic_tradingview_event_base): + order_bid_spread: Optional[Dec] = None + order_ask_spread: Optional[Dec] = None + order_amount: Optional[Dec] = None + order_levels: Optional[Dec] = None + order_level_spread: Optional[Dec] = None + + class tradingview_event(db_model_base, base_model): + __name__ = 'tradingview_event' __tablename__ = 'tradingview_events' id = Column(BigInteger, primary_key=True, index=True, unique=True, autoincrement=True) timestamp_received = Column(BigInteger, index=True) timestamp_event = Column(BigInteger, index=True) - event_descriptor = Column(String(100), index=True) + topic = Column(String(220), index=True) command = Column(String(100), index=True) exchange = Column(String(90), index=True) symbol = Column(String(30), index=True) @@ -50,7 +78,7 @@ class tradingview_event(db_model_base, def __init__(self, from_json=None, timestamp=None, - event_descriptor=None, + topic=None, command=None, exchange=None, symbol=None, @@ -66,7 +94,7 @@ def __init__(self, ): self.timestamp_received = int(time.time() * 1000) self.timestamp_event = timestamp - self.event_descriptor = event_descriptor + self.topic = topic self.command = command self.exchange = exchange self.symbol = symbol @@ -93,32 +121,25 @@ def __init__(self, self.order_amount = decimal_or_none(from_json.get("order_amount")) self.order_levels = decimal_or_none(from_json.get("order_levels")) self.order_level_spread = decimal_or_none(from_json.get("order_level_spread")) - self.event_descriptor = from_json.get("event_descriptor") - self._get_data_fields(from_json) - - def _get_data_fields(self, from_json): - if not self.event_descriptor: - for field in Config.tradingview_descriptor_fields: - if field in from_json: - self.event_descriptor = from_json.get(field) - break + self.topic = from_json.get("topic") async def process_event(self): logger.info(f"Received event from TradingView: {self.to_dict}") await self.db_save() - await ws_queue.broadcast(self) + mqtt_queue.broadcast(self) - async def process_event_ws(self): - logger.info(f"Received event via websocket: {self.to_dict}") - await self.db_save() - await ws_queue.broadcast(self) + @property + def to_pydantic(self): + if Config.include_extra_order_fields: + return pydantic_tradingview_event_with_order.from_orm(self) + return pydantic_tradingview_event_base.from_orm(self) @property def to_dict(self): the_dict = { "timestamp_received": self.timestamp_received, "timestamp_event": self.timestamp_event, - "event_descriptor": self.event_descriptor, + "topic": self.topic, "command": self.command, "exchange": self.exchange, "symbol": self.symbol, @@ -135,23 +156,9 @@ def to_dict(self): the_dict["order_level_spread"] = self.order_level_spread return the_dict - @property - def to_json(self): - return ujson.dumps(self.to_dict) - - @classmethod - def from_json(cls, - data, - raw=False): - json_data = ujson.loads(data) if raw else data - new_event = cls(from_json=json_data) - if new_event.event_descriptor: - return new_event - return None - @classmethod async def fetch_latest(cls, - event_descriptor=None): + topic=None): result = None async with AsyncSession(db.engine, expire_on_commit=False) as session: @@ -159,8 +166,8 @@ async def fetch_latest(cls, stmt = (select(cls) .limit(1) .order_by(cls.timestamp_received.desc())) - if event_descriptor: - stmt = stmt.where(cls.event_descriptor == event_descriptor) + if topic: + stmt = stmt.where(cls.topic == topic) result = (await session.execute(stmt)).fetchone() if result: result = result[0] diff --git a/rogerthat/models/web_request.py b/rogerthat/models/web_request.py index 24a7e59..5605038 100644 --- a/rogerthat/models/web_request.py +++ b/rogerthat/models/web_request.py @@ -74,17 +74,10 @@ def check_valid_api_key_tv(self): "api_key" in self._request_args_keys and self._request_args_data["api_key"] in Config.api_allowed_keys_tv) - def check_valid_api_key_hbot(self): - return (self._request_args_keys and - "api_key" in self._request_args_keys and - self._request_args_data["api_key"] in Config.api_allowed_keys_hbot) - def check_valid_json(self): - return (self._json_data and - any(k in Config.tradingview_descriptor_fields for k in list(self._json_data.keys()))) + return (self._json_data and 'topic' in list(self._json_data.keys())) async def check_is_valid(self, - for_hbot_api=False, for_tv_api=False,): if for_tv_api and not self.check_valid_user_agent(): logger.warning("Invalid User Agent detected.") @@ -100,9 +93,6 @@ async def check_is_valid(self, logger.warning("Invalid api key detected.") await self.log_request_full() return False - if for_hbot_api and not self.check_valid_api_key_hbot(): - logger.warning("Invalid api key detected.") - return False if for_tv_api and not self._json_data: await self.build_json_data() if not for_tv_api or self.check_valid_json(): diff --git a/rogerthat/models/wss_request.py b/rogerthat/models/wss_request.py deleted file mode 100644 index 852c0c7..0000000 --- a/rogerthat/models/wss_request.py +++ /dev/null @@ -1,70 +0,0 @@ -import asyncio -from rogerthat.config.config import Config -from rogerthat.db.models.tradingview_event import tradingview_event -from rogerthat.logging.configure import AsyncioLogger - - -logger = AsyncioLogger.get_logger_main(__name__) - - -class wss_request: - def __init__(self, - from_quart=None, - ws_queue=None): - self._quart_request = None - self._auth = None - self._headers = None - self._user_agent = None - self._ws_queue = ws_queue - self._received_events = set() - if from_quart: - self._quart_request = from_quart - self._auth = self._quart_request.authorization - self._headers = self._quart_request.headers - self._user_agent = self._quart_request.user_agent.string.lower().strip() - - def _check_api_key(self): - if self._headers and self._headers.get("HBOT-API-KEY") in Config.api_allowed_keys_hbot: - return True - return False - - def _check_user_agent(self): - if self._user_agent and self._user_agent == "hummingbot": - return True - return False - - def check_auth(self): - if Config.disable_websocket_authentication: - return True - return all([ - self._check_api_key(), - self._check_user_agent(), - ]) - - async def sending(self): - while True: - await self._quart_request.send(await self._ws_queue.get()) - - async def receiving(self): - while True: - data = await self._quart_request.receive() - try: - remote_event = tradingview_event.from_json(data, raw=True) - if remote_event: - event_id = f"{remote_event.timestamp_received}{remote_event.timestamp_event}" - if event_id in self._received_events: - continue - await remote_event.process_event_ws() - except Exception as e: - logger.error(f"Websocket data received: {data}") - logger.error(f"Websocket receive error: {e}") - - async def process_wss(self, tv_event=None): - producer = asyncio.create_task(self.sending()) - consumer = asyncio.create_task(self.receiving()) - if tv_event and Config.rebroadcast_on_ws_connect: - await self._ws_queue.put(tv_event.to_json) - return await asyncio.gather(producer, consumer) - - def __repr__(self): - return f"{vars(self)}" diff --git a/rogerthat/mqtt/messages.py b/rogerthat/mqtt/messages.py new file mode 100644 index 0000000..9baa65a --- /dev/null +++ b/rogerthat/mqtt/messages.py @@ -0,0 +1,113 @@ +from typing import Any, List, Optional, Tuple + +from commlib.msg import PubSubMessage, RPCMessage + + +class MQTT_STATUS_CODE: + ERROR: int = 400 + SUCCESS: int = 200 + + +class NotifyMessage(PubSubMessage): + seq: Optional[int] = 0 + timestamp: Optional[int] = -1 + msg: Optional[str] = '' + + +class EventMessage(PubSubMessage): + timestamp: Optional[int] = -1 + type: Optional[str] = 'Unknown' + data: Optional[dict] = {} + + +class LogMessage(PubSubMessage): + timestamp: float = 0.0 + msg: str = '' + level_no: int = 0 + level_name: str = '' + logger_name: str = '' + + +class StartCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + log_level: Optional[str] = None + restore: Optional[bool] = False + script: Optional[str] = None + is_quickstart: Optional[bool] = False + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + + +class StopCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + skip_order_cancellation: Optional[bool] = False + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + + +class ConfigCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + params: Optional[List[Tuple[str, Any]]] = [] + + class Response(RPCMessage.Response): + changes: Optional[List[Tuple[str, Any]]] = [] + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + + +class ImportCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + strategy: str + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + + +class StatusCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + pass + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + data: Optional[str] = '' + + +class HistoryCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + days: Optional[float] = 0 + verbose: Optional[bool] = False + precision: Optional[int] = None + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + trades: Optional[List[Any]] = [] + + +class BalanceLimitCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + exchange: str + asset: str + amount: float + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + data: Optional[str] = '' + + +class BalancePaperCommandMessage(RPCMessage): + class Request(RPCMessage.Request): + asset: str + amount: float + + class Response(RPCMessage.Response): + status: Optional[int] = MQTT_STATUS_CODE.SUCCESS + msg: Optional[str] = '' + data: Optional[str] = '' diff --git a/rogerthat/mqtt/mqtt.py b/rogerthat/mqtt/mqtt.py new file mode 100644 index 0000000..ecf5897 --- /dev/null +++ b/rogerthat/mqtt/mqtt.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python + +from typing import TYPE_CHECKING + +from commlib.node import Node +from commlib.transports.mqtt import ConnectionParameters as MQTTConnectionParameters +from rogerthat.config.config import Config +from rogerthat.logging.configure import AsyncioLogger +from rogerthat.mqtt.messages import ( + EventMessage, +) + +if TYPE_CHECKING: + from rogerthat.db.models.tradingview_event import tradingview_event + + +logger = AsyncioLogger.get_logger_main(__name__) + + +class MQTTPublisher: + + def __init__(self, + topic: str, + mqtt_node: Node): + + self._node = mqtt_node + + self._topic = topic + + self.publisher = self._node.create_publisher( + topic=self._topic, msg_type=EventMessage + ) + + def broadcast(self, event: "tradingview_event"): + self.publisher.publish(event) + + +class MQTTGateway(Node): + NODE_NAME = 'rogerthat.$UID' + HEARTBEAT_URI = 'rogerthat/$UID/hb' + + def __init__(self, + *args, **kwargs): + self.mqtt_publisher = None + + self.HEARTBEAT_URI = self.HEARTBEAT_URI.replace('$UID', Config.mqtt_instance_name) + + self._params = self._create_mqtt_params_from_conf() + + self._topic_publishers = {} + + super().__init__( + node_name=self.NODE_NAME.replace('$UID', Config.mqtt_instance_name), + connection_params=self._params, + heartbeat_uri=self.HEARTBEAT_URI, + debug=True, + *args, + **kwargs + ) + + def get_publisher_for(self, topic: str): + if topic not in self._topic_publishers: + logger.info(f"Starting MQTT Publisher for {topic}") + self._topic_publishers[topic] = MQTTPublisher(topic=topic, mqtt_node=self) + return self._topic_publishers[topic] + + def _create_mqtt_params_from_conf(self): + return MQTTConnectionParameters( + host=Config.mqtt_host, + port=int(Config.mqtt_port), + username=Config.mqtt_username, + password=Config.mqtt_password, + ssl=Config.mqtt_ssl + ) diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py new file mode 100644 index 0000000..434afb3 --- /dev/null +++ b/rogerthat/queues/mqtt_queue.py @@ -0,0 +1,61 @@ +import asyncio +from ssl import SSLCertVerificationError, SSLEOFError +from rogerthat.config.config import Config +from rogerthat.logging.configure import AsyncioLogger +from rogerthat.mqtt.mqtt import MQTTGateway +from rogerthat.utils.asyncio_tasks import safe_ensure_future + + +logger = AsyncioLogger.get_logger_main(__name__) + + +class mqtt_queue_cls: + def __init__(self): + self._mqtt_queue_task = None + self._mqtt_queue = None + self._mqtt: MQTTGateway = None + self._failure_msg = "Failed to connect to MQTT Broker!" + + if Config.mqtt_enable: + try: + self._mqtt = MQTTGateway() + self._mqtt + self._mqtt.run() + self.start() + except ConnectionRefusedError: + logger.error(f"{self._failure_msg} Check host and port!") + except SSLEOFError: + logger.error(f"{self._failure_msg} Using plain HTTP port with SSL enabled!") + except SSLCertVerificationError: + logger.error(f"{self._failure_msg} You need to set up your SSL certificates correctly!") + except Exception as e: + logger.error(e) + raise e + logger.info("MQTT Gateway is ready.") + + def _create_queue(self): + self._mqtt_queue = asyncio.Queue() + + def start(self): + if self._mqtt: + self._create_queue() + self._mqtt_queue_task = safe_ensure_future( + self._listen_for_broadcasts() + ) + + async def _listen_for_broadcasts(self): + if not self._mqtt: + raise Exception("listen_for_broadcasts called but mqtt is not enabled!") + while True: + msg = await self._mqtt_queue.get() + print("Got msg") + publisher = self._mqtt.get_publisher_for(msg.topic) + publisher.broadcast(msg.to_pydantic) + + def broadcast(self, event): + if self._mqtt: + logger.info("Broadcasting message to mqtt queue.") + self._mqtt_queue.put_nowait(event) + + +mqtt_queue = mqtt_queue_cls() diff --git a/rogerthat/queues/ws_queue.py b/rogerthat/queues/ws_queue.py deleted file mode 100644 index 02d31dd..0000000 --- a/rogerthat/queues/ws_queue.py +++ /dev/null @@ -1,67 +0,0 @@ -import asyncio -from functools import wraps -from rogerthat.utils.regexes import regexes -from rogerthat.logging.configure import AsyncioLogger - - -logger = AsyncioLogger.get_logger_main(__name__) - - -class websockets_queue: - def __init__(self): - self._connected_websockets = set() - self._connected_websockets_filtered = {} - - def _add_connection(self, filtered=None): - queue = asyncio.Queue() - if filtered: - if filtered not in self._get_filter_keys(): - self._connected_websockets_filtered[filtered] = set() - self._connected_websockets_filtered[filtered].add(queue) - else: - self._connected_websockets.add(queue) - return queue - - def _remove_connection(self, queue, filtered=None): - if filtered: - self._connected_websockets_filtered[filtered].remove(queue) - else: - self._connected_websockets.remove(queue) - - def _get_filter_keys(self): - return list(self._connected_websockets_filtered.keys()) - - def _is_filtered_event(self, event): - filter_keys = self._get_filter_keys() - return event.event_descriptor and event.event_descriptor in filter_keys - - def _get_filter_from_args(self, kwargs): - filter_name = None - if kwargs: - channel_name = kwargs.get("channel") - if channel_name and regexes.valid_ws_channel_name.match(channel_name): - filter_name = channel_name - return filter_name - - def collect_websocket(self, func): - @wraps(func) - async def wrapper(*args, **kwargs): - filter_name = self._get_filter_from_args(kwargs) - queue = self._add_connection(filter_name) - try: - return await func(queue, *args, **kwargs) - finally: - self._remove_connection(queue, filter_name) - return wrapper - - async def broadcast(self, event): - logger.info("Broadcasting message to all websocket queues.") - for queue in self._connected_websockets: - await queue.put(event.to_json) - if self._is_filtered_event(event): - logger.info(f"Broadcasting message to {event.event_descriptor} websocket queue.") - for queue in self._connected_websockets_filtered[event.event_descriptor]: - await queue.put(event.to_json) - - -ws_queue = websockets_queue() diff --git a/rogerthat/route_handlers/handlers/hummingbot.py b/rogerthat/route_handlers/handlers/hummingbot.py deleted file mode 100644 index 239af02..0000000 --- a/rogerthat/route_handlers/handlers/hummingbot.py +++ /dev/null @@ -1,44 +0,0 @@ -from quart import ( - abort, - jsonify, -) -from rogerthat.models.web_request import web_request -from rogerthat.models.wss_request import wss_request -from rogerthat.db.models.tradingview_event import tradingview_event -from rogerthat.logging.configure import AsyncioLogger - - -logger = AsyncioLogger.get_logger_main(__name__) - - -class route_handlers_hummingbot: - def __init__(self): - pass - - async def route_handler_hummingbot(self, quart_request): - request = web_request(from_quart=quart_request) - await request.build_request_args() - valid_request = await request.check_is_valid(for_hbot_api=True) - if valid_request: - latest_event = await tradingview_event.fetch_latest() - if latest_event: - return jsonify(latest_event.to_dict) - return jsonify(None) - return abort(401) - - -class wss_handlers_hummingbot: - def __init__(self): - pass - - async def wss_handler_hummingbot(self, ws_request, ws_queue, ws_channel=None): - request = wss_request(from_quart=ws_request, ws_queue=ws_queue) - valid_request = request.check_auth() - if valid_request: - if ws_channel: - logger.info(f"New websocket client connected on channel: {ws_channel}") - else: - logger.info("New websocket client connected.") - latest_event = await tradingview_event.fetch_latest(event_descriptor=ws_channel) - await request.process_wss(latest_event) - return abort(401) diff --git a/rogerthat/route_handlers/route_handlers_ctrl.py b/rogerthat/route_handlers/route_handlers_ctrl.py index 7f1c1c2..cf47285 100644 --- a/rogerthat/route_handlers/route_handlers_ctrl.py +++ b/rogerthat/route_handlers/route_handlers_ctrl.py @@ -1,14 +1,9 @@ from rogerthat.route_handlers.handlers.main import route_handlers_main -from rogerthat.route_handlers.handlers.hummingbot import ( - wss_handlers_hummingbot, - route_handlers_hummingbot, -) + from rogerthat.route_handlers.handlers.tradingview import route_handlers_tradingview -class route_handlers_ctrl(wss_handlers_hummingbot, - route_handlers_main, - route_handlers_hummingbot, +class route_handlers_ctrl(route_handlers_main, route_handlers_tradingview): pass diff --git a/rogerthat/server/routes.py b/rogerthat/server/routes.py index 939c4cd..3ea4da8 100644 --- a/rogerthat/server/routes.py +++ b/rogerthat/server/routes.py @@ -2,11 +2,10 @@ Blueprint, make_response, request, - websocket, ) from rogerthat.config.config import Config from rogerthat.route_handlers.route_handlers_ctrl import route_handlers -from rogerthat.queues.ws_queue import ws_queue +# from rogerthat.queues.ws_queue import ws_queue routes_main = Blueprint('routes_main', __name__) @@ -22,25 +21,3 @@ async def api_route_tv_webhook(): response = await make_response((await route_handlers.route_handler_tradingview_webhook(request)), 200) # response.timeout = Config.api_response_timeout return response - - -# @routes_main.before_app_websocket -# def before(): -# print("before") - - -@routes_main.route(f"/{Config.web_root}/hbot/", methods=['GET']) -async def api_route_hummingbot(): - return await make_response((await route_handlers.route_handler_hummingbot(request)), 200) - - -@routes_main.websocket(f"/{Config.wss_root}") -@ws_queue.collect_websocket -async def api_wss_hbot(queue): - return await route_handlers.wss_handler_hummingbot(websocket, queue) - - -@routes_main.websocket(f"/{Config.wss_root}/") -@ws_queue.collect_websocket -async def api_wss_hbot_filtered(queue, channel): - return await route_handlers.wss_handler_hummingbot(websocket, queue, channel) diff --git a/rogerthat/utils/splash.py b/rogerthat/utils/splash.py index 65ec83c..9411db3 100644 --- a/rogerthat/utils/splash.py +++ b/rogerthat/utils/splash.py @@ -10,7 +10,7 @@ ======================================================================================= Welcome to Roger That, a mini web server designed to receive TradingView webhook alerts -(or similar) and forward them to Hummingbot via websockets or REST queries. +(or similar) and forward them to Hummingbot via the MQTT Bridge. Server is now listening and serving your every command. diff --git a/scripts/setup.py b/scripts/setup.py index a2e1812..842beb0 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -19,14 +19,8 @@ def parse_args(): help="Update hostname.") parser.add_argument('--generate-api-key-tv', '-t', dest="generate_api_key_tv", action='store_true', help="Generate and save a new TradingView api key to the config.") - parser.add_argument('--generate-api-key-hbot', '-b', dest="generate_api_key_hbot", action='store_true', - help="Generate and save a new Hummingbot api key to the config.") parser.add_argument('--generate-quart-secrets', '-q', dest="generate_quart_secrets", action='store_true', help="Generate and save new quart secrets.") - parser.add_argument('--enable-websocket-auth', dest="enable_websocket_auth", action='store_true', - help="Enable websockets authentication.") - parser.add_argument('--disable-websocket-auth', dest="disable_websocket_auth", action='store_true', - help="Disable websockets authentication.") parser.add_argument('--enable-iptables-cloudflare', dest="enable_iptables", action='store_true', help="Enable iptables firewall rules to only allow Cloudflare traffic.") parser.add_argument('--disable-iptables-cloudflare', dest="disable_iptables", action='store_true', @@ -50,18 +44,12 @@ def parse_args(): if args.generate_api_key_tv: config_utils.check_configs() config_utils.save_new_api_key_tv() - if args.generate_api_key_hbot: - config_utils.check_configs() - config_utils.save_new_api_key_hbot() if args.generate_quart_secrets: config_utils.check_configs() config_utils.generate_quart_secrets() if args.hostname: config_utils.check_configs() config_utils.save_new_hostname(args.hostname) - if args.enable_websocket_auth or args.disable_websocket_auth: - config_utils.check_configs() - config_utils.toggle_websocket_auth(disable=True if args.disable_websocket_auth else False) if args.enable_iptables or args.disable_iptables: config_utils.check_configs() config_utils.toggle_iptables(enable=True if args.enable_iptables else False) diff --git a/support/environment.yml b/support/environment.yml index 74ce091..76842d3 100644 --- a/support/environment.yml +++ b/support/environment.yml @@ -6,6 +6,7 @@ dependencies: - python=3.8 - pip=21.2.4 - psycopg2 + - pydantic=1.9 - pip: - aiofiles==0.7.0 - alembic==1.7.7 @@ -18,3 +19,4 @@ dependencies: - sqlalchemy==1.4.32 - ujson==4.1.0 - websockets==10.2 + - git+https://github.com/robotics-4-all/commlib-py.git@f4d11c37f4aaaa20600e748fcb09bf3487f057d1 diff --git a/tests/test_websocket.py b/tests/test_websocket.py deleted file mode 100755 index 81297ba..0000000 --- a/tests/test_websocket.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -import ujson -import websockets -import path_util # noqa: F401 -from rogerthat.config.config import Config - - -async def test_websocket(): - ws_url = f"ws://localhost:{Config.quart_server_port}/{Config.wss_root}" - headers = { - "HBOT-API-KEY": Config.api_allowed_keys_hbot[0], - "User-Agent": "hummingbot", - } - ws_client = await websockets.connect(ws_url, extra_headers=headers) - print("\nConnected to websocket\n\n") - try: - while True: - raw_msg_str: str = await asyncio.wait_for(ws_client.recv(), timeout=10000) - json_msg = ujson.loads(raw_msg_str) - print(json_msg) - finally: - print("\n\nClosing websocket.\n") - if ws_client: - await ws_client.close() - - -try: - asyncio.run(test_websocket()) -except KeyboardInterrupt: - quit() diff --git a/tests/test_websocket_filtered.py b/tests/test_websocket_filtered.py deleted file mode 100644 index 22017f3..0000000 --- a/tests/test_websocket_filtered.py +++ /dev/null @@ -1,30 +0,0 @@ -import asyncio -import ujson -import websockets -import path_util # noqa: F401 -from rogerthat.config.config import Config - - -async def test_websocket(): - ws_url = f"ws://localhost:{Config.quart_server_port}/{Config.wss_root}/hummingbot_1" - headers = { - "HBOT-API-KEY": Config.api_allowed_keys_hbot[0], - "User-Agent": "hummingbot", - } - ws_client = await websockets.connect(ws_url, extra_headers=headers) - print("\nConnected to websocket\n\n") - try: - while True: - raw_msg_str: str = await asyncio.wait_for(ws_client.recv(), timeout=10000) - json_msg = ujson.loads(raw_msg_str) - print(json_msg) - finally: - print("\n\nClosing websocket.\n") - if ws_client: - await ws_client.close() - - -try: - asyncio.run(test_websocket()) -except KeyboardInterrupt: - quit() From 93980895f920ebe0298d6111ac2a89a019579734 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sun, 25 Dec 2022 04:06:30 +0000 Subject: [PATCH 02/31] MQTT Broadcast fixes --- ...70319712f5b_add_amount_and_asset_fields.py | 34 +++++ rogerthat/db/models/tradingview_event.py | 124 +++++++++++------- rogerthat/queues/mqtt_queue.py | 6 +- 3 files changed, 114 insertions(+), 50 deletions(-) create mode 100644 alembic/versions/d70319712f5b_add_amount_and_asset_fields.py diff --git a/alembic/versions/d70319712f5b_add_amount_and_asset_fields.py b/alembic/versions/d70319712f5b_add_amount_and_asset_fields.py new file mode 100644 index 0000000..eb3c46a --- /dev/null +++ b/alembic/versions/d70319712f5b_add_amount_and_asset_fields.py @@ -0,0 +1,34 @@ +"""Add amount and asset fields + +Revision ID: d70319712f5b +Revises: b492ff9dde0a +Create Date: 2022-12-25 03:58:56.086134+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd70319712f5b' +down_revision = 'b492ff9dde0a' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tradingview_events', sa.Column('asset', sa.String(length=30), nullable=True)) + op.add_column('tradingview_events', sa.Column('amount', sa.Numeric(), nullable=True)) + op.create_index(op.f('ix_tradingview_events_amount'), 'tradingview_events', ['amount'], unique=False) + op.create_index(op.f('ix_tradingview_events_asset'), 'tradingview_events', ['asset'], unique=False) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_tradingview_events_asset'), table_name='tradingview_events') + op.drop_index(op.f('ix_tradingview_events_amount'), table_name='tradingview_events') + op.drop_column('tradingview_events', 'amount') + op.drop_column('tradingview_events', 'asset') + # ### end Alembic commands ### diff --git a/rogerthat/db/models/tradingview_event.py b/rogerthat/db/models/tradingview_event.py index 994d735..2dc9338 100644 --- a/rogerthat/db/models/tradingview_event.py +++ b/rogerthat/db/models/tradingview_event.py @@ -1,5 +1,6 @@ from decimal import Decimal as Dec import time +from pydantic import create_model from commlib.msg import PubSubMessage from sqlalchemy import ( Column, @@ -7,7 +8,6 @@ Numeric, String, ) -from typing import Optional from rogerthat.config.config import Config from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.future import select @@ -29,29 +29,8 @@ logger = AsyncioLogger.get_logger_main(__name__) -class pydantic_tradingview_event_base(PubSubMessage): - id: int - timestamp_received: int - timestamp_event: Optional[int] = None - topic: str - command: Optional[str] = None - exchange: Optional[str] = None - symbol: Optional[str] = None - interval: Optional[int] = None - price: Optional[Dec] = None - volume: Optional[Dec] = None - inventory: Optional[Dec] = None - - class Config: - orm_mode = True - - -class pydantic_tradingview_event_with_order(pydantic_tradingview_event_base): - order_bid_spread: Optional[Dec] = None - order_ask_spread: Optional[Dec] = None - order_amount: Optional[Dec] = None - order_levels: Optional[Dec] = None - order_level_spread: Optional[Dec] = None +class pydantic_tradingview_event(PubSubMessage): + pass class tradingview_event(db_model_base, @@ -65,9 +44,11 @@ class tradingview_event(db_model_base, command = Column(String(100), index=True) exchange = Column(String(90), index=True) symbol = Column(String(30), index=True) + asset = Column(String(30), index=True) interval = Column(BigInteger, index=True) price = Column(Numeric, index=True) volume = Column(Numeric, index=True) + amount = Column(Numeric, index=True) inventory = Column(Numeric, index=True) order_bid_spread = Column(Numeric, index=True) order_ask_spread = Column(Numeric, index=True) @@ -82,9 +63,11 @@ def __init__(self, command=None, exchange=None, symbol=None, + asset=None, interval=None, price=Dec("0"), volume=Dec("0"), + amount=Dec("0"), inventory=Dec("0"), order_bid_spread=None, order_ask_spread=None, @@ -98,23 +81,37 @@ def __init__(self, self.command = command self.exchange = exchange self.symbol = symbol + self.asset = asset self.interval = interval self.price = price self.volume = volume + self.amount = amount self.inventory = inventory self.order_bid_spread = order_bid_spread self.order_ask_spread = order_ask_spread self.order_amount = order_amount self.order_levels = order_levels self.order_level_spread = order_level_spread + self.log_level = None + self.restore = None + self.script = None + self.is_quickstart = None + self.skip_order_cancellation = None + self.params = None + self.strategy = None + self.days = None + self.verbose = None + self.precision = None if from_json: self.timestamp_event = int_or_none(from_json.get("timestamp")) self.command = from_json.get("command") self.exchange = from_json.get("exchange") self.symbol = from_json.get("symbol") + self.asset = from_json.get("asset") self.interval = int_or_none(from_json.get("interval")) self.price = decimal_or_zero(from_json.get("price")) self.volume = decimal_or_zero(from_json.get("volume")) + self.amount = decimal_or_zero(from_json.get("amount")) self.inventory = decimal_or_zero(from_json.get("inventory")) self.order_bid_spread = decimal_or_none(from_json.get("order_bid_spread")) self.order_ask_spread = decimal_or_none(from_json.get("order_ask_spread")) @@ -122,39 +119,70 @@ def __init__(self, self.order_levels = decimal_or_none(from_json.get("order_levels")) self.order_level_spread = decimal_or_none(from_json.get("order_level_spread")) self.topic = from_json.get("topic") + self.log_level = from_json.get("log_level") + self.restore = from_json.get("restore") + self.script = from_json.get("script") + self.is_quickstart = from_json.get("is_quickstart") + self.skip_order_cancellation = from_json.get("skip_order_cancellation") + self.params = from_json.get("params") + self.strategy = from_json.get("strategy") + self.days = from_json.get("days") + self.verbose = from_json.get("verbose") + self.precision = from_json.get("precision") async def process_event(self): - logger.info(f"Received event from TradingView: {self.to_dict}") + logger.info(f"Received event from TradingView: {self.to_minimised_dict()}") await self.db_save() mqtt_queue.broadcast(self) @property def to_pydantic(self): - if Config.include_extra_order_fields: - return pydantic_tradingview_event_with_order.from_orm(self) - return pydantic_tradingview_event_base.from_orm(self) + pydantic_model = create_model("pydantic_tradingview_event", + **self.to_minimised_dict(mqtt=True), + __base__=pydantic_tradingview_event) + return pydantic_model() - @property - def to_dict(self): - the_dict = { - "timestamp_received": self.timestamp_received, - "timestamp_event": self.timestamp_event, - "topic": self.topic, - "command": self.command, - "exchange": self.exchange, - "symbol": self.symbol, - "interval": self.interval, - "price": self.price, - "volume": self.volume, - "inventory": self.inventory, - } + def to_minimised_dict(self, mqtt=False): + obj_keys = [ + "timestamp_received", + "timestamp_event", + "command", + "exchange", + "symbol", + "asset", + "interval", + "price", + "volume", + "amount", + "inventory", + "log_level", + "restore", + "script", + "is_quickstart", + "skip_order_cancellation", + "params", + "strategy", + "days", + "verbose", + "precision", + ] + if not mqtt: + obj_keys.insert(0, "topic") if Config.include_extra_order_fields: - the_dict["order_bid_spread"] = self.order_bid_spread - the_dict["order_ask_spread"] = self.order_ask_spread - the_dict["order_amount"] = self.order_amount - the_dict["order_levels"] = self.order_levels - the_dict["order_level_spread"] = self.order_level_spread - return the_dict + obj_keys.extend([ + "order_bid_spread", + "order_ask_spread", + "order_amount", + "order_levels", + "order_level_spread", + ]) + minimised_dict = {} + for key in obj_keys: + val = getattr(self, key) + if val: + minimised_dict[key] = val + + return minimised_dict @classmethod async def fetch_latest(cls, diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py index 434afb3..d174777 100644 --- a/rogerthat/queues/mqtt_queue.py +++ b/rogerthat/queues/mqtt_queue.py @@ -1,4 +1,5 @@ import asyncio +from socket import gaierror from ssl import SSLCertVerificationError, SSLEOFError from rogerthat.config.config import Config from rogerthat.logging.configure import AsyncioLogger @@ -22,7 +23,7 @@ def __init__(self): self._mqtt self._mqtt.run() self.start() - except ConnectionRefusedError: + except (ConnectionRefusedError, gaierror): logger.error(f"{self._failure_msg} Check host and port!") except SSLEOFError: logger.error(f"{self._failure_msg} Using plain HTTP port with SSL enabled!") @@ -48,11 +49,12 @@ async def _listen_for_broadcasts(self): raise Exception("listen_for_broadcasts called but mqtt is not enabled!") while True: msg = await self._mqtt_queue.get() - print("Got msg") publisher = self._mqtt.get_publisher_for(msg.topic) publisher.broadcast(msg.to_pydantic) def broadcast(self, event): + if not self._mqtt_queue: + logger.error("Cannot broadcast, MQTT not started!") if self._mqtt: logger.info("Broadcasting message to mqtt queue.") self._mqtt_queue.put_nowait(event) From dbfbe70c358c14744609eb48b72ed2f1d09fe048 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 29 Dec 2022 02:37:28 +0000 Subject: [PATCH 03/31] Refactoring, tidying up, core event changes --- .gitattributes | 2 +- Dockerfile | 2 +- alembic/env.py | 24 +- .../d40b06f2cc82_more_mqtt_refactoring.py | 105 +++++++ {tests => examples}/path_util.py | 3 +- examples/test_tradingview_alerts.py | 37 +++ rogerthat/app/delegate.py | 9 +- rogerthat/app/rogerthat.py | 29 +- rogerthat/config/config.py | 13 +- .../config/samples/main_config.sample.yml | 4 - .../config/samples/tradingview.sample.yml | 14 +- rogerthat/db/database_init.py | 26 +- rogerthat/db/engine.py | 27 +- rogerthat/db/fetch_alembic_revision.py | 7 +- rogerthat/db/models/base.py | 12 +- rogerthat/db/models/tradingview_event.py | 264 +++++++++--------- rogerthat/logging/configure.py | 4 +- rogerthat/models/web_request.py | 11 +- rogerthat/mqtt/messages.py | 114 +------- rogerthat/mqtt/mqtt.py | 30 +- rogerthat/queues/mqtt_queue.py | 22 +- rogerthat/queues/request_processing_queue.py | 51 ++++ .../route_handlers/handlers/tradingview.py | 12 +- .../route_handlers/route_handlers_ctrl.py | 14 +- rogerthat/server/routes.py | 18 +- rogerthat/server/server.py | 41 ++- rogerthat/utils/asyncio_tasks.py | 15 +- rogerthat/utils/class_helpers.py | 8 + rogerthat/utils/instance_wrapper.py | 14 +- rogerthat/utils/parsing_numbers.py | 10 +- rogerthat/utils/time_date.py | 2 +- support/environment.yml | 1 + 32 files changed, 530 insertions(+), 415 deletions(-) create mode 100644 alembic/versions/d40b06f2cc82_more_mqtt_refactoring.py rename {tests => examples}/path_util.py (99%) create mode 100644 examples/test_tradingview_alerts.py create mode 100644 rogerthat/queues/request_processing_queue.py diff --git a/.gitattributes b/.gitattributes index b825662..5a4e203 100644 --- a/.gitattributes +++ b/.gitattributes @@ -4,4 +4,4 @@ rogerthat/* eol=lf scripts/* eol=lf scripts/*.bat eol=crlf support/* eol=lf -tests/* eol=lf \ No newline at end of file +examples/* eol=lf \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index f45f05c..7f251aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -39,7 +39,7 @@ COPY --chown=rogerthat:rogerthat alembic/ alembic/ COPY --chown=rogerthat:rogerthat bin/ bin/ COPY --chown=rogerthat:rogerthat rogerthat/ rogerthat/ COPY --chown=rogerthat:rogerthat scripts/ scripts/ -COPY --chown=rogerthat:rogerthat tests/ tests/ +COPY --chown=rogerthat:rogerthat examples/ examples/ COPY --chown=rogerthat:rogerthat alembic.ini . COPY --chown=rogerthat:rogerthat support/docker_compose_entrypoint.sh . COPY --chown=rogerthat:rogerthat support/docker_start_python.sh . diff --git a/alembic/env.py b/alembic/env.py index 465282d..f086970 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -1,27 +1,27 @@ # Logging # from logging.config import fileConfig # SqlAlchemy -from sqlalchemy import engine_from_config -from sqlalchemy import pool -# Alembic -from alembic import context # Add parent to path import sys from pathlib import Path + +from sqlalchemy import engine_from_config, pool + +# Alembic +from alembic import context + sys.path.append(str(Path('.').absolute())) -from rogerthat.utils import path_import # noqa: F401, E402 -from rogerthat.db.models import ( # noqa: E402 - base_model, -) from rogerthat.config.config import Config # noqa: E402 +from rogerthat.db.models import base_model # noqa: E402 +from rogerthat.utils import path_import # noqa: F401, E402 # this is the Alembic Config object, which provides # access to the values within the .ini file in use. alembic_config = context.config -DB_URL = "postgresql://{0}:{1}@{2}/{3}".format(Config.database_user, - Config.database_password, - Config.database_host, - Config.database_name) +DB_URL = "postgresql://{0}:{1}@{2}/{3}".format(Config.get_inst().database_user, + Config.get_inst().database_password, + Config.get_inst().database_host, + Config.get_inst().database_name) alembic_config.set_main_option('sqlalchemy.url', DB_URL) # Interpret the config file for Python logging. diff --git a/alembic/versions/d40b06f2cc82_more_mqtt_refactoring.py b/alembic/versions/d40b06f2cc82_more_mqtt_refactoring.py new file mode 100644 index 0000000..8f1b7a0 --- /dev/null +++ b/alembic/versions/d40b06f2cc82_more_mqtt_refactoring.py @@ -0,0 +1,105 @@ +"""more mqtt refactoring + +Revision ID: d40b06f2cc82 +Revises: d70319712f5b +Create Date: 2022-12-29 03:29:15.752967+00:00 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'd40b06f2cc82' +down_revision = 'd70319712f5b' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tradingview_events', sa.Column('sequence', sa.BigInteger(), nullable=True)) + op.add_column('tradingview_events', sa.Column('event_type', sa.String(length=90), nullable=True)) + op.add_column('tradingview_events', sa.Column('params', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('tradingview_events', sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=True)) + op.add_column('tradingview_events', sa.Column('msg', sa.String(length=500), nullable=True)) + op.add_column('tradingview_events', sa.Column('log_level', sa.String(length=30), nullable=True)) + op.add_column('tradingview_events', sa.Column('restore', sa.Boolean(), nullable=True)) + op.add_column('tradingview_events', sa.Column('script', sa.String(length=90), nullable=True)) + op.add_column('tradingview_events', sa.Column('is_quickstart', sa.Boolean(), nullable=True)) + op.add_column('tradingview_events', sa.Column('skip_order_cancellation', sa.Boolean(), nullable=True)) + op.add_column('tradingview_events', sa.Column('strategy', sa.String(length=90), nullable=True)) + op.add_column('tradingview_events', sa.Column('days', sa.Numeric(), nullable=True)) + op.add_column('tradingview_events', sa.Column('verbose', sa.Boolean(), nullable=True)) + op.add_column('tradingview_events', sa.Column('precision', sa.Integer(), nullable=True)) + op.drop_index('ix_tradingview_events_command', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_interval', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_inventory', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_order_amount', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_order_ask_spread', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_order_bid_spread', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_order_level_spread', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_order_levels', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_price', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_symbol', table_name='tradingview_events') + op.drop_index('ix_tradingview_events_volume', table_name='tradingview_events') + op.create_index(op.f('ix_tradingview_events_event_type'), 'tradingview_events', ['event_type'], unique=False) + op.create_index(op.f('ix_tradingview_events_sequence'), 'tradingview_events', ['sequence'], unique=False) + op.create_index(op.f('ix_tradingview_events_strategy'), 'tradingview_events', ['strategy'], unique=False) + op.drop_column('tradingview_events', 'interval') + op.drop_column('tradingview_events', 'order_bid_spread') + op.drop_column('tradingview_events', 'inventory') + op.drop_column('tradingview_events', 'volume') + op.drop_column('tradingview_events', 'order_ask_spread') + op.drop_column('tradingview_events', 'price') + op.drop_column('tradingview_events', 'order_level_spread') + op.drop_column('tradingview_events', 'symbol') + op.drop_column('tradingview_events', 'order_levels') + op.drop_column('tradingview_events', 'command') + op.drop_column('tradingview_events', 'order_amount') + op.execute("delete from tradingview_events") + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tradingview_events', sa.Column('order_amount', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('command', sa.VARCHAR(length=100), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('order_levels', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('symbol', sa.VARCHAR(length=30), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('order_level_spread', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('price', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('order_ask_spread', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('volume', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('inventory', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('order_bid_spread', sa.NUMERIC(), autoincrement=False, nullable=True)) + op.add_column('tradingview_events', sa.Column('interval', sa.BIGINT(), autoincrement=False, nullable=True)) + op.drop_index(op.f('ix_tradingview_events_strategy'), table_name='tradingview_events') + op.drop_index(op.f('ix_tradingview_events_sequence'), table_name='tradingview_events') + op.drop_index(op.f('ix_tradingview_events_event_type'), table_name='tradingview_events') + op.create_index('ix_tradingview_events_volume', 'tradingview_events', ['volume'], unique=False) + op.create_index('ix_tradingview_events_symbol', 'tradingview_events', ['symbol'], unique=False) + op.create_index('ix_tradingview_events_price', 'tradingview_events', ['price'], unique=False) + op.create_index('ix_tradingview_events_order_levels', 'tradingview_events', ['order_levels'], unique=False) + op.create_index('ix_tradingview_events_order_level_spread', 'tradingview_events', ['order_level_spread'], unique=False) + op.create_index('ix_tradingview_events_order_bid_spread', 'tradingview_events', ['order_bid_spread'], unique=False) + op.create_index('ix_tradingview_events_order_ask_spread', 'tradingview_events', ['order_ask_spread'], unique=False) + op.create_index('ix_tradingview_events_order_amount', 'tradingview_events', ['order_amount'], unique=False) + op.create_index('ix_tradingview_events_inventory', 'tradingview_events', ['inventory'], unique=False) + op.create_index('ix_tradingview_events_interval', 'tradingview_events', ['interval'], unique=False) + op.create_index('ix_tradingview_events_command', 'tradingview_events', ['command'], unique=False) + op.drop_column('tradingview_events', 'precision') + op.drop_column('tradingview_events', 'verbose') + op.drop_column('tradingview_events', 'days') + op.drop_column('tradingview_events', 'strategy') + op.drop_column('tradingview_events', 'skip_order_cancellation') + op.drop_column('tradingview_events', 'is_quickstart') + op.drop_column('tradingview_events', 'script') + op.drop_column('tradingview_events', 'restore') + op.drop_column('tradingview_events', 'log_level') + op.drop_column('tradingview_events', 'msg') + op.drop_column('tradingview_events', 'data') + op.drop_column('tradingview_events', 'params') + op.drop_column('tradingview_events', 'event_type') + op.drop_column('tradingview_events', 'sequence') + # ### end Alembic commands ### diff --git a/tests/path_util.py b/examples/path_util.py similarity index 99% rename from tests/path_util.py rename to examples/path_util.py index 12e2aba..895e3cf 100644 --- a/tests/path_util.py +++ b/examples/path_util.py @@ -1,3 +1,4 @@ -from os.path import dirname, realpath import sys +from os.path import dirname, realpath + sys.path.insert(0, dirname(dirname(realpath(__file__)))) diff --git a/examples/test_tradingview_alerts.py b/examples/test_tradingview_alerts.py new file mode 100644 index 0000000..3d5e9aa --- /dev/null +++ b/examples/test_tradingview_alerts.py @@ -0,0 +1,37 @@ +import asyncio +import sys + +import aiohttp +import path_util # noqa: F401 + +try: + from rogerthat.config.config import Config +except Exception: + print("Unable to load Config for tests, edit example manually.") + sys.exit(0) + + +test_data = { + "topic": "hbot/1/start", + "log_level": "DEBUG" +} + +ROGERTHAT_HOST = "localhost" +ROGERTHAT_PORT = Config.get_inst().quart_server_port +ROGERTHAT_API = Config.get_inst().api_allowed_keys_tv[0] + + +async def main(): + """ + Use this script to test/fake a TradingView alert. + """ + headers = { + "User-Agent": "go-http-client/1.1" + } + url = f"http://{ROGERTHAT_HOST}:{ROGERTHAT_PORT}/api/tv_webhook/?api_key={ROGERTHAT_API}" + async with aiohttp.ClientSession(headers=headers) as session: + async with session.post(url, json=test_data) as resp: + print(f"HTTP Status: {resp.status}") + print(f"Response: {await resp.text()}") + +asyncio.run(main()) diff --git a/rogerthat/app/delegate.py b/rogerthat/app/delegate.py index 696d04e..572f523 100644 --- a/rogerthat/app/delegate.py +++ b/rogerthat/app/delegate.py @@ -1,10 +1,5 @@ from rogerthat.utils.instance_wrapper import instance_wrapper -class AppWrap(instance_wrapper): - @property - def Main(self): - return self.instance - - -App = AppWrap() +class App(instance_wrapper): + pass diff --git a/rogerthat/app/rogerthat.py b/rogerthat/app/rogerthat.py index ff0c397..18edaf8 100644 --- a/rogerthat/app/rogerthat.py +++ b/rogerthat/app/rogerthat.py @@ -1,13 +1,15 @@ import asyncio import signal + from rogerthat.app.delegate import App from rogerthat.config.config import Config from rogerthat.db.database_init import database_init from rogerthat.logging.configure import AsyncioLogger +from rogerthat.queues.mqtt_queue import mqtt_queue +from rogerthat.queues.request_processing_queue import request_processing_queue from rogerthat.server.server import quart_server from rogerthat.utils.splash import splash_msg - logger = AsyncioLogger.get_logger_main(__name__) @@ -27,6 +29,8 @@ async def Initialise(self): logger.info(splash_msg) def start_server(self): + logger.info("RogerThat startup.") + def _signal_handler(*_): # noqa: N803 logger.info("Shutdown signal handler called.") self.shutdown_event.set() @@ -37,13 +41,20 @@ def _signal_handler(*_): # noqa: N803 except NotImplementedError: signal.signal(signal.SIGTERM, _signal_handler) - logger.info("Started RogerThat Server") - if Config.debug_mode: - quart_server.run(host="0.0.0.0", port=Config.quart_server_port, debug=Config.debug_mode) + request_processing_queue.get_instance() + logger.info("Starting Broadcast Queues.") + mqtt_queue.get_instance() + logger.info("Starting RogerThat Server.") + if Config.get_inst().debug_mode: + quart_server.run( + host="0.0.0.0", + port=Config.get_inst().quart_server_port, + debug=Config.get_inst().debug_mode) else: - serv_task = quart_server.run_task(host="0.0.0.0", - port=Config.quart_server_port, - use_reloader=False, - shutdown_trigger=self.shutdown_event.wait) + serv_task = quart_server.run_task( + host="0.0.0.0", + port=Config.get_inst().quart_server_port, + use_reloader=False, + shutdown_trigger=self.shutdown_event.wait) loop.run_until_complete(serv_task) - logger.info("Stopped RogerThat Server") + logger.info("Stopped RogerThat Server.") diff --git a/rogerthat/config/config.py b/rogerthat/config/config.py index affd4c4..46fb4fc 100644 --- a/rogerthat/config/config.py +++ b/rogerthat/config/config.py @@ -1,8 +1,8 @@ # Private config vars import os -from rogerthat.utils.class_helpers import no_setters -from rogerthat.config.loader import config_loader +from rogerthat.config.loader import config_loader +from rogerthat.utils.class_helpers import no_setters _loaded_configs = config_loader() _app_config = _loaded_configs.app_config @@ -12,12 +12,11 @@ _web_config = _loaded_configs.web_config -class ConfigSetup(no_setters): +class Config(no_setters): # Main App _project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) _app_name = _app_config['app_name'] _debug_mode = _app_config['debug_mode'] - _include_extra_order_fields = _app_config.get('include_extra_order_fields', False) # Web _server_host = _web_config['server_host'] @@ -51,7 +50,5 @@ class ConfigSetup(no_setters): _api_allowed_keys_tv = _web_config['api_allowed_keys_tv'] # Trading View - _tradingview_translations = _tv_config['tradingview_translations'] - - -Config = ConfigSetup() + _tradingview_include_extra_fields = _tv_config.get('tradingview_include_extra_fields') + _tradingview_exclude_fields = _tv_config.get('tradingview_exclude_fields') diff --git a/rogerthat/config/samples/main_config.sample.yml b/rogerthat/config/samples/main_config.sample.yml index 7670994..381f36d 100644 --- a/rogerthat/config/samples/main_config.sample.yml +++ b/rogerthat/config/samples/main_config.sample.yml @@ -7,9 +7,5 @@ template_version: 0.2 app_name: rogerthat # ADVANCED! -# Setting this option to `true` will include extra order fields in sent data. -# These fields won't do anything without your intervention, they require scripting of Hummingbot. -include_extra_order_fields: false - # Enabled debug mode debug_mode: false diff --git a/rogerthat/config/samples/tradingview.sample.yml b/rogerthat/config/samples/tradingview.sample.yml index 661c7e9..473ba91 100644 --- a/rogerthat/config/samples/tradingview.sample.yml +++ b/rogerthat/config/samples/tradingview.sample.yml @@ -1,10 +1,12 @@ ######################################### ### TradingView config ### ######################################### -template_version: 0.2 +template_version: 0.3 -# A list of key names to use for the event descriptor on received events. -# Only one will be used. -# Event data must contain a descriptor key to be accepted. -tradingview_translations: - name: translated_name +# A list of fields or key names to be included in event broadcasts +tradingview_include_extra_fields: + - extra_data + +# A list of fields or key names to be excluded from event broadcasts +tradingview_exclude_fields: + - exclude_this_key diff --git a/rogerthat/db/database_init.py b/rogerthat/db/database_init.py index a31bee7..20dca81 100644 --- a/rogerthat/db/database_init.py +++ b/rogerthat/db/database_init.py @@ -1,25 +1,25 @@ import asyncio # noqa: F401 import os from socket import gaierror + from sqlalchemy import text + +from alembic import command as alembic_cmd + # Alembic from alembic.config import Config as alembic_config -from alembic import command as alembic_cmd from rogerthat.config.config import Config -from rogerthat.db.models import ( - base_model, -) -from rogerthat.db.engine import db +from rogerthat.db.engine import db_engine from rogerthat.db.fetch_alembic_revision import fetch_alembic_revision +from rogerthat.db.models import base_model from rogerthat.logging.configure import AsyncioLogger - logger = AsyncioLogger.get_logger_db(__name__) class database_init(): _meta = base_model.metadata - _alembic_cfg = alembic_config(os.path.join(Config.project_root, 'alembic.ini')) + _alembic_cfg = alembic_config(os.path.join(Config.get_inst().project_root, 'alembic.ini')) # ************************************************************************************************* # @@ -29,14 +29,14 @@ class database_init(): @classmethod async def create_db(cls): logger.info("Database does not exist yet, creating.") - async with db.engine_root.connect() as conn: - await conn.execute(text(f"CREATE DATABASE {Config.database_name}")) + async with db_engine.db().engine_root.connect() as conn: + await conn.execute(text(f"CREATE DATABASE {Config.get_inst().database_name}")) return True @classmethod async def create_tables(cls): logger.info("Creating new or missing db tables.") - async with db.engine.begin() as conn: + async with db_engine.db().engine.begin() as conn: await conn.run_sync(cls._meta.create_all) logger.info("Done creating tables.") return True @@ -51,8 +51,8 @@ async def initialise(cls): try: await cls.create_db() await cls.create_tables() - except gaierror: - logger.error("Failed to connect to SQL database.") + except (ConnectionRefusedError, gaierror, OSError): + logger.error("Failed to connect to SQL database. Check host and port.") return None logger.info("Checking alembic.") @@ -73,7 +73,7 @@ async def initialise(cls): # """ # Completely wipe and recreate all tables # """ - # async with db.engine.begin() as conn: + # async with db_engine.db().engine.begin() as conn: # await conn.run_sync(cls._meta.drop_all) # await conn.run_sync(cls._meta.create_all, checkfirst=False) # return True diff --git a/rogerthat/db/engine.py b/rogerthat/db/engine.py index 68894c4..5128772 100644 --- a/rogerthat/db/engine.py +++ b/rogerthat/db/engine.py @@ -1,8 +1,23 @@ from sqlalchemy.ext.asyncio import create_async_engine + from rogerthat.config.config import Config class db_engine(): + _shared_instance: "db_engine" = None + + @classmethod + def db(cls) -> "db_engine": + if cls._shared_instance is None: + cls._shared_instance = cls( + db_name=Config.get_inst().database_name, + db_root_user=Config.get_inst().database_user_root, + db_user=Config.get_inst().database_user, + db_pw=Config.get_inst().database_password, + protocol=Config.get_inst().database_protocol, + db_host=Config.get_inst().database_host) + return cls._shared_instance + def __init__(self, db_name, db_root_user='postgres', @@ -13,9 +28,9 @@ def __init__(self, self._db_url_main = f"{protocol}://{db_user}:{db_pw}@{db_host}/{db_name}" self._db_url_root = f"{protocol}://{db_user}:{db_pw}@{db_host}/{db_root_user}" self._engine_main = create_async_engine(self._db_url_main, - echo=Config.debug_mode) + echo=Config.get_inst().debug_mode) self._engine_root = create_async_engine(self._db_url_root, - echo=Config.debug_mode, + echo=Config.get_inst().debug_mode, isolation_level='AUTOCOMMIT') @property @@ -25,11 +40,3 @@ def engine(self): @property def engine_root(self): return self._engine_root - - -db = db_engine(db_name=Config.database_name, - db_root_user=Config.database_user_root, - db_user=Config.database_user, - db_pw=Config.database_password, - protocol=Config.database_protocol, - db_host=Config.database_host) diff --git a/rogerthat/db/fetch_alembic_revision.py b/rogerthat/db/fetch_alembic_revision.py index 2344318..a8d75cd 100644 --- a/rogerthat/db/fetch_alembic_revision.py +++ b/rogerthat/db/fetch_alembic_revision.py @@ -1,12 +1,13 @@ -from alembic.migration import MigrationContext from sqlalchemy import create_engine + +from alembic.migration import MigrationContext from rogerthat.config.config import Config def fetch_alembic_revision(): - url = (f"postgresql://{Config.database_user}:{Config.database_password}@" - f"{Config.database_host}/{Config.database_name}") + url = (f"postgresql://{Config.get_inst().database_user}:{Config.get_inst().database_password}@" + f"{Config.get_inst().database_host}/{Config.get_inst().database_name}") engine = create_engine(url) conn = engine.connect() diff --git a/rogerthat/db/models/base.py b/rogerthat/db/models/base.py index 3a9aded..00260bf 100644 --- a/rogerthat/db/models/base.py +++ b/rogerthat/db/models/base.py @@ -1,10 +1,8 @@ from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.ext.declarative import ( - declarative_base, -) +from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.future import select -from rogerthat.db.engine import db +from rogerthat.db.engine import db_engine # Setup SQL BaseModel base_model = declarative_base() @@ -23,7 +21,7 @@ async def db_save(self): Save one. """ result = None - async with AsyncSession(db.engine, + async with AsyncSession(db_engine.db().engine, expire_on_commit=False) as session: async with session.begin(): session.add_all([self]) @@ -36,7 +34,7 @@ async def db_find(cls, """ Find one object by ID. """ - async with AsyncSession(db.engine, + async with AsyncSession(db_engine.db().engine, expire_on_commit=False) as session: async with session.begin(): if ByID is not None: @@ -47,7 +45,7 @@ async def db_find(cls, return result async def db_delete(self): - async with AsyncSession(db.engine, + async with AsyncSession(db_engine.db().engine, expire_on_commit=False) as session: async with session.begin(): await session.delete(self) diff --git a/rogerthat/db/models/tradingview_event.py b/rogerthat/db/models/tradingview_event.py index 2dc9338..2270d6c 100644 --- a/rogerthat/db/models/tradingview_event.py +++ b/rogerthat/db/models/tradingview_event.py @@ -1,36 +1,69 @@ -from decimal import Decimal as Dec import time + from pydantic import create_model -from commlib.msg import PubSubMessage -from sqlalchemy import ( - Column, - BigInteger, - Numeric, - String, -) -from rogerthat.config.config import Config +from sqlalchemy import BigInteger, Boolean, Column, Integer, Numeric, String +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.ext.mutable import MutableDict from sqlalchemy.future import select -from rogerthat.db.engine import db -from rogerthat.db.models.base import ( - base_model, - db_model_base, -) -from rogerthat.queues.mqtt_queue import mqtt_queue -# from rogerthat.queues.ws_queue import ws_queue -from rogerthat.utils.parsing_numbers import ( - decimal_or_none, - decimal_or_zero, - int_or_none, -) -from rogerthat.logging.configure import AsyncioLogger +from rogerthat.config.config import Config +from rogerthat.db.engine import db_engine +from rogerthat.db.models.base import base_model, db_model_base +from rogerthat.logging.configure import AsyncioLogger +from rogerthat.mqtt.messages import TradingviewMessage +from rogerthat.queues.mqtt_queue import mqtt_queue +from rogerthat.utils.parsing_numbers import bool_or_none, decimal_or_none, int_or_none logger = AsyncioLogger.get_logger_main(__name__) -class pydantic_tradingview_event(PubSubMessage): - pass +EVENT_KEYS = [ + "topic", + "timestamp_received", + "timestamp_event", + "sequence", + "event_type", + "params", + "data", + "msg", + "exchange", + "asset", + "amount", + "log_level", + "restore", + "script", + "is_quickstart", + "skip_order_cancellation", + "strategy", + "days", + "verbose", + "precision", +] + + +class filtered_event_keys: + _filtered = None + _key_translations = { + "event_type": "type", + } + + @classmethod + def translate(cls, evt_key): + return cls._key_translations.get(evt_key, evt_key) + + @classmethod + def list(cls): + if cls._filtered is None: + if Config.get_inst().tradingview_include_extra_fields: + cls._filtered = list(dict.fromkeys((EVENT_KEYS + Config.get_inst().tradingview_include_extra_fields))) + else: + cls._filtered = list(EVENT_KEYS) + if Config.get_inst().tradingview_exclude_fields: + for excl_key in Config.get_inst().tradingview_exclude_fields: + if excl_key != "topic" and excl_key in cls._filtered: + cls._filtered.remove(excl_key) + return cls._filtered class tradingview_event(db_model_base, @@ -38,149 +71,116 @@ class tradingview_event(db_model_base, __name__ = 'tradingview_event' __tablename__ = 'tradingview_events' id = Column(BigInteger, primary_key=True, index=True, unique=True, autoincrement=True) + topic = Column(String(220), index=True) timestamp_received = Column(BigInteger, index=True) timestamp_event = Column(BigInteger, index=True) - topic = Column(String(220), index=True) - command = Column(String(100), index=True) + sequence = Column(BigInteger, index=True) + event_type = Column(String(90), index=True) + params = Column(MutableDict.as_mutable(JSONB)) + data = Column(MutableDict.as_mutable(JSONB)) + msg = Column(String(500)) exchange = Column(String(90), index=True) - symbol = Column(String(30), index=True) asset = Column(String(30), index=True) - interval = Column(BigInteger, index=True) - price = Column(Numeric, index=True) - volume = Column(Numeric, index=True) amount = Column(Numeric, index=True) - inventory = Column(Numeric, index=True) - order_bid_spread = Column(Numeric, index=True) - order_ask_spread = Column(Numeric, index=True) - order_amount = Column(Numeric, index=True) - order_levels = Column(Numeric, index=True) - order_level_spread = Column(Numeric, index=True) + log_level = Column(String(30)) + restore = Column(Boolean) + script = Column(String(90)) + is_quickstart = Column(Boolean) + skip_order_cancellation = Column(Boolean) + strategy = Column(String(90), index=True) + days = Column(Numeric) + verbose = Column(Boolean) + precision = Column(Integer) def __init__(self, from_json=None, - timestamp=None, topic=None, - command=None, + timestamp=None, + sequence=None, + event_type=None, + params=None, + data=None, + msg=None, exchange=None, - symbol=None, asset=None, - interval=None, - price=Dec("0"), - volume=Dec("0"), - amount=Dec("0"), - inventory=Dec("0"), - order_bid_spread=None, - order_ask_spread=None, - order_amount=None, - order_levels=None, - order_level_spread=None, + amount=None, + log_level=None, + restore=None, + script=None, + is_quickstart=None, + skip_order_cancellation=None, + strategy=None, + days=None, + verbose=None, + precision=None, ): + self.topic = topic self.timestamp_received = int(time.time() * 1000) self.timestamp_event = timestamp - self.topic = topic - self.command = command + self.sequence = sequence + self.event_type = event_type + self.params = params + self.data = data + self.msg = msg self.exchange = exchange - self.symbol = symbol self.asset = asset - self.interval = interval - self.price = price - self.volume = volume self.amount = amount - self.inventory = inventory - self.order_bid_spread = order_bid_spread - self.order_ask_spread = order_ask_spread - self.order_amount = order_amount - self.order_levels = order_levels - self.order_level_spread = order_level_spread - self.log_level = None - self.restore = None - self.script = None - self.is_quickstart = None - self.skip_order_cancellation = None - self.params = None - self.strategy = None - self.days = None - self.verbose = None - self.precision = None + self.log_level = log_level + self.restore = restore + self.script = script + self.is_quickstart = is_quickstart + self.skip_order_cancellation = skip_order_cancellation + self.strategy = strategy + self.days = days + self.verbose = verbose + self.precision = precision + if Config.get_inst().tradingview_include_extra_fields: + for evt_key in Config.get_inst().tradingview_include_extra_fields: + setattr(self, evt_key, None) if from_json: + self.topic = from_json.get("topic") self.timestamp_event = int_or_none(from_json.get("timestamp")) - self.command = from_json.get("command") + self.sequence = int_or_none(from_json.get("sequence")) + self.event_type = from_json.get("type") + self.params = from_json.get("params") + self.data = from_json.get("data") + self.msg = from_json.get("msg") self.exchange = from_json.get("exchange") - self.symbol = from_json.get("symbol") self.asset = from_json.get("asset") - self.interval = int_or_none(from_json.get("interval")) - self.price = decimal_or_zero(from_json.get("price")) - self.volume = decimal_or_zero(from_json.get("volume")) - self.amount = decimal_or_zero(from_json.get("amount")) - self.inventory = decimal_or_zero(from_json.get("inventory")) - self.order_bid_spread = decimal_or_none(from_json.get("order_bid_spread")) - self.order_ask_spread = decimal_or_none(from_json.get("order_ask_spread")) - self.order_amount = decimal_or_none(from_json.get("order_amount")) - self.order_levels = decimal_or_none(from_json.get("order_levels")) - self.order_level_spread = decimal_or_none(from_json.get("order_level_spread")) - self.topic = from_json.get("topic") + self.amount = decimal_or_none(from_json.get("amount")) self.log_level = from_json.get("log_level") - self.restore = from_json.get("restore") + self.restore = bool_or_none(from_json.get("restore")) self.script = from_json.get("script") - self.is_quickstart = from_json.get("is_quickstart") - self.skip_order_cancellation = from_json.get("skip_order_cancellation") - self.params = from_json.get("params") + self.is_quickstart = bool_or_none(from_json.get("is_quickstart")) + self.skip_order_cancellation = bool_or_none(from_json.get("skip_order_cancellation")) self.strategy = from_json.get("strategy") - self.days = from_json.get("days") - self.verbose = from_json.get("verbose") - self.precision = from_json.get("precision") + self.days = decimal_or_none(from_json.get("days")) + self.verbose = bool_or_none(from_json.get("verbose")) + self.precision = int_or_none(from_json.get("precision")) + if Config.get_inst().tradingview_include_extra_fields: + for evt_key in Config.get_inst().tradingview_include_extra_fields: + setattr(self, evt_key, from_json.get(evt_key)) async def process_event(self): - logger.info(f"Received event from TradingView: {self.to_minimised_dict()}") + logger.debug(f"Processing event from TradingView: {self.to_minimised_dict()}") await self.db_save() - mqtt_queue.broadcast(self) + mqtt_queue.get_instance().broadcast(self) @property def to_pydantic(self): - pydantic_model = create_model("pydantic_tradingview_event", + pydantic_model = create_model("TradingviewMessage", **self.to_minimised_dict(mqtt=True), - __base__=pydantic_tradingview_event) + __base__=TradingviewMessage) return pydantic_model() def to_minimised_dict(self, mqtt=False): - obj_keys = [ - "timestamp_received", - "timestamp_event", - "command", - "exchange", - "symbol", - "asset", - "interval", - "price", - "volume", - "amount", - "inventory", - "log_level", - "restore", - "script", - "is_quickstart", - "skip_order_cancellation", - "params", - "strategy", - "days", - "verbose", - "precision", - ] - if not mqtt: - obj_keys.insert(0, "topic") - if Config.include_extra_order_fields: - obj_keys.extend([ - "order_bid_spread", - "order_ask_spread", - "order_amount", - "order_levels", - "order_level_spread", - ]) minimised_dict = {} - for key in obj_keys: + for i, key in enumerate(filtered_event_keys.list()): + if i == 0 and mqtt: + continue val = getattr(self, key) - if val: - minimised_dict[key] = val + if val is not None: + minimised_dict[filtered_event_keys.translate(key)] = val return minimised_dict @@ -188,7 +188,7 @@ def to_minimised_dict(self, mqtt=False): async def fetch_latest(cls, topic=None): result = None - async with AsyncSession(db.engine, + async with AsyncSession(db_engine.db().engine, expire_on_commit=False) as session: async with session.begin(): stmt = (select(cls) diff --git a/rogerthat/logging/configure.py b/rogerthat/logging/configure.py index 7dbcbbe..70df8f4 100644 --- a/rogerthat/logging/configure.py +++ b/rogerthat/logging/configure.py @@ -4,10 +4,10 @@ from os.path import join as path_join from queue import SimpleQueue as Queue from typing import List + from rogerthat.config.config import Config from rogerthat.logging.colours import ColouredFormatter - LOGGING_FILE_LIMIT = 1e6 LOG_LEVEL_ROOT = logging.DEBUG LOG_LEVEL_GENERAL = logging.DEBUG @@ -66,7 +66,7 @@ def _build_logging_handler_file(cls, log_file, filters=None): if log_file in cls._file_handlers: return file_formatter = logging.Formatter('[%(asctime)s] %(name)s - %(levelname)s - %(message)s') - log_path = path_join(Config.project_root, "logs", f"{Config.app_name}-{log_file}.log") + log_path = path_join(Config.get_inst().project_root, "logs", f"{Config.get_inst().app_name}-{log_file}.log") file_handler = AsyncLoggingRotatingFileHandler( log_path, "a", diff --git a/rogerthat/models/web_request.py b/rogerthat/models/web_request.py index 5605038..4c8d570 100644 --- a/rogerthat/models/web_request.py +++ b/rogerthat/models/web_request.py @@ -1,7 +1,6 @@ from rogerthat.config.config import Config from rogerthat.logging.configure import AsyncioLogger - logger = AsyncioLogger.get_logger_main(__name__) @@ -63,7 +62,7 @@ async def build_json_data(self): return True def check_valid_user_agent(self): - return self._user_agent and self._user_agent in Config.accepted_user_agents_tv + return self._user_agent and self._user_agent in Config.get_inst().accepted_user_agents_tv def check_valid_content_type(self): return self._content_type and (self._content_type.startswith('application/json') or @@ -72,10 +71,14 @@ def check_valid_content_type(self): def check_valid_api_key_tv(self): return (self._request_args_keys and "api_key" in self._request_args_keys and - self._request_args_data["api_key"] in Config.api_allowed_keys_tv) + self._request_args_data["api_key"] in Config.get_inst().api_allowed_keys_tv) def check_valid_json(self): - return (self._json_data and 'topic' in list(self._json_data.keys())) + if not self._json_data: + return False + if isinstance(self._json_data, list): + return all(['topic' in list(jdata.keys()) for jdata in self._json_data]) + return 'topic' in list(self._json_data.keys()) async def check_is_valid(self, for_tv_api=False,): diff --git a/rogerthat/mqtt/messages.py b/rogerthat/mqtt/messages.py index 9baa65a..05a9ff4 100644 --- a/rogerthat/mqtt/messages.py +++ b/rogerthat/mqtt/messages.py @@ -1,113 +1,5 @@ -from typing import Any, List, Optional, Tuple +from commlib.msg import PubSubMessage -from commlib.msg import PubSubMessage, RPCMessage - -class MQTT_STATUS_CODE: - ERROR: int = 400 - SUCCESS: int = 200 - - -class NotifyMessage(PubSubMessage): - seq: Optional[int] = 0 - timestamp: Optional[int] = -1 - msg: Optional[str] = '' - - -class EventMessage(PubSubMessage): - timestamp: Optional[int] = -1 - type: Optional[str] = 'Unknown' - data: Optional[dict] = {} - - -class LogMessage(PubSubMessage): - timestamp: float = 0.0 - msg: str = '' - level_no: int = 0 - level_name: str = '' - logger_name: str = '' - - -class StartCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - log_level: Optional[str] = None - restore: Optional[bool] = False - script: Optional[str] = None - is_quickstart: Optional[bool] = False - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - - -class StopCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - skip_order_cancellation: Optional[bool] = False - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - - -class ConfigCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - params: Optional[List[Tuple[str, Any]]] = [] - - class Response(RPCMessage.Response): - changes: Optional[List[Tuple[str, Any]]] = [] - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - - -class ImportCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - strategy: str - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - - -class StatusCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - pass - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - data: Optional[str] = '' - - -class HistoryCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - days: Optional[float] = 0 - verbose: Optional[bool] = False - precision: Optional[int] = None - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - trades: Optional[List[Any]] = [] - - -class BalanceLimitCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - exchange: str - asset: str - amount: float - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - data: Optional[str] = '' - - -class BalancePaperCommandMessage(RPCMessage): - class Request(RPCMessage.Request): - asset: str - amount: float - - class Response(RPCMessage.Response): - status: Optional[int] = MQTT_STATUS_CODE.SUCCESS - msg: Optional[str] = '' - data: Optional[str] = '' +class TradingviewMessage(PubSubMessage): + pass diff --git a/rogerthat/mqtt/mqtt.py b/rogerthat/mqtt/mqtt.py index ecf5897..2618be5 100644 --- a/rogerthat/mqtt/mqtt.py +++ b/rogerthat/mqtt/mqtt.py @@ -4,11 +4,10 @@ from commlib.node import Node from commlib.transports.mqtt import ConnectionParameters as MQTTConnectionParameters + from rogerthat.config.config import Config from rogerthat.logging.configure import AsyncioLogger -from rogerthat.mqtt.messages import ( - EventMessage, -) +from rogerthat.mqtt.messages import TradingviewMessage if TYPE_CHECKING: from rogerthat.db.models.tradingview_event import tradingview_event @@ -28,29 +27,34 @@ def __init__(self, self._topic = topic self.publisher = self._node.create_publisher( - topic=self._topic, msg_type=EventMessage + topic=self._topic, msg_type=TradingviewMessage ) def broadcast(self, event: "tradingview_event"): + logger.debug(f"Broadcasting MQTT event on {self._topic}: {event}") self.publisher.publish(event) class MQTTGateway(Node): - NODE_NAME = 'rogerthat.$UID' - HEARTBEAT_URI = 'rogerthat/$UID/hb' + NODE_NAME = "$APP.$UID" + HEARTBEAT_URI = "$APP/$UID/hb" def __init__(self, *args, **kwargs): self.mqtt_publisher = None - self.HEARTBEAT_URI = self.HEARTBEAT_URI.replace('$UID', Config.mqtt_instance_name) + self.HEARTBEAT_URI = self.HEARTBEAT_URI.replace('$UID', Config.get_inst().mqtt_instance_name) + self.HEARTBEAT_URI = self.HEARTBEAT_URI.replace('$APP', Config.get_inst().app_name) + + self.NODE_NAME = self.NODE_NAME.replace('$UID', Config.get_inst().mqtt_instance_name) + self.NODE_NAME = self.NODE_NAME.replace('$APP', Config.get_inst().app_name) self._params = self._create_mqtt_params_from_conf() self._topic_publishers = {} super().__init__( - node_name=self.NODE_NAME.replace('$UID', Config.mqtt_instance_name), + node_name=self.NODE_NAME, connection_params=self._params, heartbeat_uri=self.HEARTBEAT_URI, debug=True, @@ -66,9 +70,9 @@ def get_publisher_for(self, topic: str): def _create_mqtt_params_from_conf(self): return MQTTConnectionParameters( - host=Config.mqtt_host, - port=int(Config.mqtt_port), - username=Config.mqtt_username, - password=Config.mqtt_password, - ssl=Config.mqtt_ssl + host=Config.get_inst().mqtt_host, + port=int(Config.get_inst().mqtt_port), + username=Config.get_inst().mqtt_username, + password=Config.get_inst().mqtt_password, + ssl=Config.get_inst().mqtt_ssl ) diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py index d174777..305ae9f 100644 --- a/rogerthat/queues/mqtt_queue.py +++ b/rogerthat/queues/mqtt_queue.py @@ -1,29 +1,36 @@ import asyncio from socket import gaierror from ssl import SSLCertVerificationError, SSLEOFError + from rogerthat.config.config import Config from rogerthat.logging.configure import AsyncioLogger from rogerthat.mqtt.mqtt import MQTTGateway from rogerthat.utils.asyncio_tasks import safe_ensure_future - logger = AsyncioLogger.get_logger_main(__name__) -class mqtt_queue_cls: +class mqtt_queue: + _shared_instance: "mqtt_queue" = None + + @classmethod + def get_instance(cls) -> "mqtt_queue": + if cls._shared_instance is None: + cls._shared_instance = cls() + return cls._shared_instance + def __init__(self): self._mqtt_queue_task = None self._mqtt_queue = None self._mqtt: MQTTGateway = None self._failure_msg = "Failed to connect to MQTT Broker!" - if Config.mqtt_enable: + if Config.get_inst().mqtt_enable: try: self._mqtt = MQTTGateway() - self._mqtt self._mqtt.run() self.start() - except (ConnectionRefusedError, gaierror): + except (ConnectionRefusedError, gaierror, OSError): logger.error(f"{self._failure_msg} Check host and port!") except SSLEOFError: logger.error(f"{self._failure_msg} Using plain HTTP port with SSL enabled!") @@ -56,8 +63,5 @@ def broadcast(self, event): if not self._mqtt_queue: logger.error("Cannot broadcast, MQTT not started!") if self._mqtt: - logger.info("Broadcasting message to mqtt queue.") + logger.debug("Adding event to MQTT queue.") self._mqtt_queue.put_nowait(event) - - -mqtt_queue = mqtt_queue_cls() diff --git a/rogerthat/queues/request_processing_queue.py b/rogerthat/queues/request_processing_queue.py new file mode 100644 index 0000000..12bb054 --- /dev/null +++ b/rogerthat/queues/request_processing_queue.py @@ -0,0 +1,51 @@ +import asyncio + +from rogerthat.db.models.tradingview_event import tradingview_event +from rogerthat.logging.configure import AsyncioLogger +from rogerthat.utils.asyncio_tasks import safe_ensure_future + +logger = AsyncioLogger.get_logger_main(__name__) + + +class request_processing_queue: + _shared_instance: "request_processing_queue" = None + + @classmethod + def get_instance(cls) -> "request_processing_queue": + if cls._shared_instance is None: + cls._shared_instance = cls() + return cls._shared_instance + + def __init__(self): + self._request_processing_queue_task = None + self._request_processing_queue = None + + self.start() + logger.info("Request Processing Queue ready.") + + def _create_queue(self): + self._request_processing_queue = asyncio.Queue() + + def start(self): + self._create_queue() + self._request_processing_queue_task = safe_ensure_future( + self._listen_for_requests() + ) + + async def _listen_for_requests(self): + while True: + req = await self._request_processing_queue.get() + if isinstance(req, list): + for req_msg in req: + tv_event = tradingview_event(from_json=req_msg) + safe_ensure_future(tv_event.process_event(), with_timeout=1800.0) + else: + tv_event = tradingview_event(from_json=req) + safe_ensure_future(tv_event.process_event(), with_timeout=1800.0) + + def add_request(self, request): + if not self._request_processing_queue: + logger.error("Cannot process requests, queue not started!") + return + logger.debug("Adding TradingView event to processing queue.") + self._request_processing_queue.put_nowait(request) diff --git a/rogerthat/route_handlers/handlers/tradingview.py b/rogerthat/route_handlers/handlers/tradingview.py index bc6346a..3965498 100644 --- a/rogerthat/route_handlers/handlers/tradingview.py +++ b/rogerthat/route_handlers/handlers/tradingview.py @@ -1,10 +1,7 @@ -from quart import ( - abort, - jsonify, -) +from quart import abort, jsonify + from rogerthat.models.web_request import web_request -from rogerthat.db.models.tradingview_event import tradingview_event -from rogerthat.utils.asyncio_tasks import safe_ensure_future +from rogerthat.queues.request_processing_queue import request_processing_queue class route_handlers_tradingview: @@ -16,7 +13,6 @@ async def route_handler_tradingview_webhook(self, quart_request): await request.build_request_args() valid_request = await request.check_is_valid(for_tv_api=True) if valid_request: - tv_event = tradingview_event(from_json=request.json_data) - safe_ensure_future(tv_event.process_event()) + request_processing_queue.get_instance().add_request(request.json_data) return jsonify({"success": True}) return abort(401) diff --git a/rogerthat/route_handlers/route_handlers_ctrl.py b/rogerthat/route_handlers/route_handlers_ctrl.py index cf47285..31404d1 100644 --- a/rogerthat/route_handlers/route_handlers_ctrl.py +++ b/rogerthat/route_handlers/route_handlers_ctrl.py @@ -1,11 +1,13 @@ from rogerthat.route_handlers.handlers.main import route_handlers_main - from rogerthat.route_handlers.handlers.tradingview import route_handlers_tradingview -class route_handlers_ctrl(route_handlers_main, - route_handlers_tradingview): - pass - +class route_handlers(route_handlers_main, + route_handlers_tradingview): + _shared_instance: "route_handlers" = None -route_handlers = route_handlers_ctrl() + @classmethod + def get_instance(cls) -> "route_handlers": + if cls._shared_instance is None: + cls._shared_instance = cls() + return cls._shared_instance diff --git a/rogerthat/server/routes.py b/rogerthat/server/routes.py index 3ea4da8..c7b77a1 100644 --- a/rogerthat/server/routes.py +++ b/rogerthat/server/routes.py @@ -1,10 +1,8 @@ -from quart import ( - Blueprint, - make_response, - request, -) +from quart import Blueprint, make_response, request + from rogerthat.config.config import Config from rogerthat.route_handlers.route_handlers_ctrl import route_handlers + # from rogerthat.queues.ws_queue import ws_queue @@ -13,11 +11,13 @@ @routes_main.route("/", methods=['GET']) async def main_route(): - return await make_response((await route_handlers.route_handler_main(request)), 200) + handler_data = await route_handlers.get_instance().route_handler_main(request) + return await make_response(handler_data, 200) -@routes_main.route(f"/{Config.web_root}/tv_webhook/", methods=['POST', 'GET']) +@routes_main.route(f"/{Config.get_inst().web_root}/tv_webhook/", methods=['POST', 'GET']) async def api_route_tv_webhook(): - response = await make_response((await route_handlers.route_handler_tradingview_webhook(request)), 200) - # response.timeout = Config.api_response_timeout + handler_data = await route_handlers.get_instance().route_handler_tradingview_webhook(request) + response = await make_response(handler_data, 200) + # response.timeout = Config.get_inst().api_response_timeout return response diff --git a/rogerthat/server/server.py b/rogerthat/server/server.py index aa4a54e..59a16d9 100644 --- a/rogerthat/server/server.py +++ b/rogerthat/server/server.py @@ -1,26 +1,17 @@ import asyncio import logging -from quart import ( - Quart, -) + +from quart import Quart + # from quart_session import Session -from quart_auth import ( - AuthManager, - # AuthUser, - # login_user, -) -from rogerthat.config.config import Config +from quart_auth import AuthManager # AuthUser,; login_user, + from rogerthat.app.delegate import App -from rogerthat.extensions.extension_quart_json import ( - JSONDecoder, - JSONEncoder, -) +from rogerthat.config.config import Config +from rogerthat.extensions.extension_quart_json import JSONDecoder, JSONEncoder from rogerthat.server.routes import routes_main +from rogerthat.utils.misc import set_bash_title from rogerthat.utils.time_date import time_in_seconds -from rogerthat.utils.misc import ( - set_bash_title, -) - ######################################################################### # * @@ -28,12 +19,12 @@ # * logging.getLogger('quart.app').setLevel(logging.ERROR) logging.getLogger('quart.serving').setLevel(logging.ERROR) -quart_server = Quart(f"{Config.app_name}/API") -quart_server.config["QUART_AUTH_COOKIE_DOMAIN"] = Config.server_host -if Config.debug_mode: - quart_server.config["QUART_AUTH_COOKIE_DOMAIN"] = Config.quart_cookie_domain_debug +quart_server = Quart(f"{Config.get_inst().app_name}/API") +quart_server.config["QUART_AUTH_COOKIE_DOMAIN"] = Config.get_inst().server_host +if Config.get_inst().debug_mode: + quart_server.config["QUART_AUTH_COOKIE_DOMAIN"] = Config.get_inst().quart_cookie_domain_debug quart_server.config["QUART_AUTH_COOKIE_NAME"] = "HH_AUTH" -quart_server.config["QUART_AUTH_SALT"] = Config.quart_auth_csalt +quart_server.config["QUART_AUTH_SALT"] = Config.get_inst().quart_auth_csalt quart_server.config["QUART_AUTH_COOKIE_SECURE"] = False quart_server.config["QUART_AUTH_DURATION"] = time_in_seconds.year quart_server.config["TEMPLATES_AUTO_RELOAD"] = True @@ -43,7 +34,7 @@ # Generate New Secret key via: # import secrets # secrets.token_urlsafe(16) -quart_server.secret_key = Config.quart_secret_key +quart_server.secret_key = Config.get_inst().quart_secret_key # JSON extension quart_server.json_encoder = JSONEncoder quart_server.json_decoder = JSONDecoder @@ -57,6 +48,6 @@ @quart_server.before_serving async def startup(): - set_bash_title(Config.app_name) + set_bash_title(Config.get_inst().app_name) loop = asyncio.get_event_loop() - loop.create_task(App.Main.Initialise()) + loop.create_task(App.get_instance().Initialise()) diff --git a/rogerthat/utils/asyncio_tasks.py b/rogerthat/utils/asyncio_tasks.py index 2079b95..9b6da95 100644 --- a/rogerthat/utils/asyncio_tasks.py +++ b/rogerthat/utils/asyncio_tasks.py @@ -1,16 +1,19 @@ import asyncio -import time import inspect +import time import traceback -from rogerthat.logging.configure import AsyncioLogger +from rogerthat.logging.configure import AsyncioLogger logger = AsyncioLogger.get_logger_main(__name__) -async def safe_wrapper(c): +async def safe_wrapper(c, with_timeout=None): try: - return await asyncio.wait_for(c, timeout=1800.0) + if with_timeout: + return await asyncio.wait_for(c, timeout=with_timeout) + else: + return await c except asyncio.CancelledError: raise except Exception as e: @@ -18,8 +21,8 @@ async def safe_wrapper(c): logger.error(f"Unhandled error in background task: {str(e)}\n{tb}") -def safe_ensure_future(coro, *args, **kwargs): - return asyncio.ensure_future(safe_wrapper(coro), *args, **kwargs) +def safe_ensure_future(coro, with_timeout=None, *args, **kwargs): + return asyncio.ensure_future(safe_wrapper(coro, with_timeout=with_timeout), *args, **kwargs) async def safe_gather(*args, **kwargs): diff --git a/rogerthat/utils/class_helpers.py b/rogerthat/utils/class_helpers.py index 8b147ac..b8ebfaf 100644 --- a/rogerthat/utils/class_helpers.py +++ b/rogerthat/utils/class_helpers.py @@ -1,4 +1,12 @@ class no_setters(object): + _shared_instance: "no_setters" = None + + @classmethod + def get_inst(cls) -> "no_setters": + if cls._shared_instance is None: + cls._shared_instance = cls() + return cls._shared_instance + def __init__(self): pass diff --git a/rogerthat/utils/instance_wrapper.py b/rogerthat/utils/instance_wrapper.py index 822298f..eebee94 100644 --- a/rogerthat/utils/instance_wrapper.py +++ b/rogerthat/utils/instance_wrapper.py @@ -1,6 +1,12 @@ class instance_wrapper: - def __init__(self): - self.instance = None + _shared_instance: "instance_wrapper" = None - def update_instance(self, new_instance): - self.instance = new_instance + @classmethod + def update_instance(cls, new_instance): + cls._shared_instance = new_instance + + @classmethod + def get_instance(cls) -> "instance_wrapper": + if cls._shared_instance is None: + raise Exception("instance_wrapper instance has not been set yet!") + return cls._shared_instance diff --git a/rogerthat/utils/parsing_numbers.py b/rogerthat/utils/parsing_numbers.py index 9c267d6..89c9fbb 100644 --- a/rogerthat/utils/parsing_numbers.py +++ b/rogerthat/utils/parsing_numbers.py @@ -2,12 +2,16 @@ def decimal_or_none(num_string): - return Dec(str(num_string)) if num_string and str(num_string).replace('.', '').isdigit() else None + return Dec(str(num_string)) if num_string is not None and str(num_string).replace('.', '').isdigit() else None def decimal_or_zero(num_string): - return Dec(str(num_string)) if num_string and str(num_string).replace('.', '').isdigit() else Dec("0") + return Dec(str(num_string)) if num_string is not None and str(num_string).replace('.', '').isdigit() else Dec("0") def int_or_none(num_string): - return int(str(num_string)) if num_string and str(num_string).isdigit() else None + return int(str(num_string)) if num_string is not None and str(num_string).isdigit() else None + + +def bool_or_none(bool_string): + return bool(bool_string) if bool_string is not None else None diff --git a/rogerthat/utils/time_date.py b/rogerthat/utils/time_date.py index 3531bbf..88a7899 100644 --- a/rogerthat/utils/time_date.py +++ b/rogerthat/utils/time_date.py @@ -11,4 +11,4 @@ class time_in_seconds_constants(no_setters): _year = int(_day * 365) -time_in_seconds = time_in_seconds_constants() +time_in_seconds = time_in_seconds_constants.get_inst() diff --git a/support/environment.yml b/support/environment.yml index 76842d3..85eaad3 100644 --- a/support/environment.yml +++ b/support/environment.yml @@ -9,6 +9,7 @@ dependencies: - pydantic=1.9 - pip: - aiofiles==0.7.0 + - aiohttp==3.8.3 - alembic==1.7.7 - asyncpg==0.25.0 - pre-commit==2.15.0 From 2dda4faa120e0134afce43cc0e06f45adfc7660f Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 29 Dec 2022 17:39:42 +0000 Subject: [PATCH 04/31] Fix timestamp_event key translation --- examples/test_tradingview_alerts.py | 36 ++++++++++++++++++++++-- rogerthat/db/models/tradingview_event.py | 1 + 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/examples/test_tradingview_alerts.py b/examples/test_tradingview_alerts.py index 3d5e9aa..926f76f 100644 --- a/examples/test_tradingview_alerts.py +++ b/examples/test_tradingview_alerts.py @@ -12,15 +12,42 @@ test_data = { - "topic": "hbot/1/start", + "topic": "hbot/hummingbot_instance_1/start", "log_level": "DEBUG" } +test_data_list = [ + { + "topic": "hbot/hummingbot_instance_1/start", + "log_level": "DEBUG" + }, + { + "topic": "hbot/hummingbot_instance_1/external/events/my_event", + "type": "external_event", + "timestamp": 1234567890, + "sequence": 1234567890, + "data": { + "exchange": "{{exchange}}", + "symbol": "{{ticker}}", + "interval": "{{interval}}", + "price": "{{close}}", + "volume": "{{volume}}", + "position": "{{strategy.market_position}}", + "inventory": "{{strategy.order.comment}}" + } + } +] + ROGERTHAT_HOST = "localhost" ROGERTHAT_PORT = Config.get_inst().quart_server_port ROGERTHAT_API = Config.get_inst().api_allowed_keys_tv[0] +async def print_response(resp): + print(f"HTTP Status: {resp.status}") + print(f"Response: {await resp.text()}") + + async def main(): """ Use this script to test/fake a TradingView alert. @@ -31,7 +58,10 @@ async def main(): url = f"http://{ROGERTHAT_HOST}:{ROGERTHAT_PORT}/api/tv_webhook/?api_key={ROGERTHAT_API}" async with aiohttp.ClientSession(headers=headers) as session: async with session.post(url, json=test_data) as resp: - print(f"HTTP Status: {resp.status}") - print(f"Response: {await resp.text()}") + await print_response(resp) + + async with aiohttp.ClientSession(headers=headers) as session: + async with session.post(url, json=test_data_list) as resp: + await print_response(resp) asyncio.run(main()) diff --git a/rogerthat/db/models/tradingview_event.py b/rogerthat/db/models/tradingview_event.py index 2270d6c..b123799 100644 --- a/rogerthat/db/models/tradingview_event.py +++ b/rogerthat/db/models/tradingview_event.py @@ -45,6 +45,7 @@ class filtered_event_keys: _filtered = None _key_translations = { + "timestamp_event": "timestamp", "event_type": "type", } From a553329fc2ad337aab7412604d3172d853b3ab04 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 29 Dec 2022 21:06:42 +0000 Subject: [PATCH 05/31] Tidy up startup/shutdown --- bin/start_rogerthat.py | 5 +- rogerthat/app/rogerthat.py | 84 ++++++++++++++------ rogerthat/queues/mqtt_queue.py | 19 ++++- rogerthat/queues/request_processing_queue.py | 27 +++++-- support/docker_compose_entrypoint.sh | 2 +- 5 files changed, 99 insertions(+), 38 deletions(-) diff --git a/bin/start_rogerthat.py b/bin/start_rogerthat.py index f58f61f..738ec81 100755 --- a/bin/start_rogerthat.py +++ b/bin/start_rogerthat.py @@ -2,9 +2,10 @@ import asyncio import os import sys + import path_util # noqa: F401 -from rogerthat.app.rogerthat import RogerThat +from rogerthat.app.rogerthat import RogerThat if __name__ == "__main__": pid = os.getpid() @@ -13,4 +14,4 @@ fp.write(f"{pid}") if sys.version_info[0] == 3 and sys.version_info[1] >= 8 and sys.platform.startswith('win'): asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - RogerThat().start_server() + RogerThat().run() diff --git a/rogerthat/app/rogerthat.py b/rogerthat/app/rogerthat.py index 18edaf8..d76e2ce 100644 --- a/rogerthat/app/rogerthat.py +++ b/rogerthat/app/rogerthat.py @@ -1,5 +1,6 @@ import asyncio import signal +import time from rogerthat.app.delegate import App from rogerthat.config.config import Config @@ -8,6 +9,7 @@ from rogerthat.queues.mqtt_queue import mqtt_queue from rogerthat.queues.request_processing_queue import request_processing_queue from rogerthat.server.server import quart_server +from rogerthat.utils.asyncio_tasks import safe_ensure_future, safe_gather from rogerthat.utils.splash import splash_msg logger = AsyncioLogger.get_logger_main(__name__) @@ -17,44 +19,76 @@ class RogerThat: def __init__(self): App.update_instance(self) self.shutdown_event = asyncio.Event() + self._request_queue = None + self._mqtt_queue = None + self._ev_loop = None + self._serv_task = None async def Initialise(self): logger.info("Initialising database.") db_started = await database_init.initialise() if not db_started: await asyncio.sleep(0.1) - self.shutdown_event.set() + self.shutdown() return logger.info("Finished initialising database.") logger.info(splash_msg) - def start_server(self): - logger.info("RogerThat startup.") - - def _signal_handler(*_): # noqa: N803 - logger.info("Shutdown signal handler called.") - self.shutdown_event.set() + def _signal_handler(self, *_): # noqa: N803 + logger.info("Shutdown signal handler called.") + self.shutdown() - loop = asyncio.get_event_loop() + def setup_loop(self): + self._ev_loop = asyncio.get_event_loop() + signals = [ + signal.SIGINT, + signal.SIGTERM, + signal.SIGQUIT, + ] try: - loop.add_signal_handler(signal.SIGTERM, _signal_handler) + for sig in signals: + self._ev_loop.add_signal_handler(sig, self._signal_handler) except NotImplementedError: - signal.signal(signal.SIGTERM, _signal_handler) + for sig in signals: + signal.signl(sig, self._signal_handler) + + def start_quart(self): + self._ev_loop.run_until_complete(self.start_quart_loop()) + + async def start_quart_loop(self): + self._serv_task = quart_server.run_task( + host="0.0.0.0", + port=Config.get_inst().quart_server_port, + debug=Config.get_inst().debug_mode, + use_reloader=False if not Config.get_inst().debug_mode else True, + shutdown_trigger=self.shutdown_event.wait) + await safe_gather(self._serv_task) + + async def exit_loop(self): + self._request_queue.stop() + self._mqtt_queue.stop() + logger.info("Stopped all queues.") + await asyncio.sleep(1) - request_processing_queue.get_instance() + def shutdown(self): + logger.info("Stopping RogerThat Server.") + self.shutdown_event.set() + safe_ensure_future(self.exit_loop(), loop=self._ev_loop) + + def start_queues(self): + self._request_queue = request_processing_queue.get_instance() logger.info("Starting Broadcast Queues.") - mqtt_queue.get_instance() + self._mqtt_queue = mqtt_queue.get_instance() + + def start_server(self): + logger.info("RogerThat startup.") + self.setup_loop() + self.start_queues() logger.info("Starting RogerThat Server.") - if Config.get_inst().debug_mode: - quart_server.run( - host="0.0.0.0", - port=Config.get_inst().quart_server_port, - debug=Config.get_inst().debug_mode) - else: - serv_task = quart_server.run_task( - host="0.0.0.0", - port=Config.get_inst().quart_server_port, - use_reloader=False, - shutdown_trigger=self.shutdown_event.wait) - loop.run_until_complete(serv_task) - logger.info("Stopped RogerThat Server.") + self.start_quart() + + def run(self): + self.start_server() + time.sleep(1) + logger.info("Stopped RogerThat Server.") + time.sleep(1) diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py index 305ae9f..6ead33e 100644 --- a/rogerthat/queues/mqtt_queue.py +++ b/rogerthat/queues/mqtt_queue.py @@ -1,4 +1,5 @@ import asyncio +import traceback from socket import gaierror from ssl import SSLCertVerificationError, SSLEOFError @@ -51,13 +52,25 @@ def start(self): self._listen_for_broadcasts() ) + def stop(self): + if self._mqtt_queue_task is not None: + self._mqtt_queue_task.cancel() + self._mqtt_queue_task = None + logger.debug("MQTT Queue stopped.") + async def _listen_for_broadcasts(self): if not self._mqtt: raise Exception("listen_for_broadcasts called but mqtt is not enabled!") while True: - msg = await self._mqtt_queue.get() - publisher = self._mqtt.get_publisher_for(msg.topic) - publisher.broadcast(msg.to_pydantic) + try: + msg = await self._mqtt_queue.get() + publisher = self._mqtt.get_publisher_for(msg.topic) + publisher.broadcast(msg.to_pydantic) + except asyncio.CancelledError: + raise + except Exception as e: + tb = "".join(traceback.TracebackException.from_exception(e).format()) + logger.error(f"Error in mqtt_queue: {e}\n{tb}") def broadcast(self, event): if not self._mqtt_queue: diff --git a/rogerthat/queues/request_processing_queue.py b/rogerthat/queues/request_processing_queue.py index 12bb054..2e1952e 100644 --- a/rogerthat/queues/request_processing_queue.py +++ b/rogerthat/queues/request_processing_queue.py @@ -1,4 +1,5 @@ import asyncio +import traceback from rogerthat.db.models.tradingview_event import tradingview_event from rogerthat.logging.configure import AsyncioLogger @@ -32,16 +33,28 @@ def start(self): self._listen_for_requests() ) + def stop(self): + if self._request_processing_queue_task is not None: + self._request_processing_queue_task.cancel() + self._request_processing_queue_task = None + logger.debug("Request Processing Queue stopped.") + async def _listen_for_requests(self): while True: - req = await self._request_processing_queue.get() - if isinstance(req, list): - for req_msg in req: - tv_event = tradingview_event(from_json=req_msg) + try: + req = await self._request_processing_queue.get() + if isinstance(req, list): + for req_msg in req: + tv_event = tradingview_event(from_json=req_msg) + safe_ensure_future(tv_event.process_event(), with_timeout=1800.0) + else: + tv_event = tradingview_event(from_json=req) safe_ensure_future(tv_event.process_event(), with_timeout=1800.0) - else: - tv_event = tradingview_event(from_json=req) - safe_ensure_future(tv_event.process_event(), with_timeout=1800.0) + except asyncio.CancelledError: + raise + except Exception as e: + tb = "".join(traceback.TracebackException.from_exception(e).format()) + logger.error(f"Error in request_processing_queue: {e}\n{tb}") def add_request(self, request): if not self._request_processing_queue: diff --git a/support/docker_compose_entrypoint.sh b/support/docker_compose_entrypoint.sh index f32e707..e98d99f 100755 --- a/support/docker_compose_entrypoint.sh +++ b/support/docker_compose_entrypoint.sh @@ -28,7 +28,7 @@ killpg(){ echo "Beginning RogerThat Shutdown." rogerthat_pid=`cat .rogerthat.pid` echo "Killing PID $rogerthat_pid." - kill "$rogerthat_pid" + kill -15 "$rogerthat_pid" while [ "$(ps -ax | grep [p]ython)" != "" ]; do sleep 1s done From d4032d727e252d62a9a4d0ed5572af3417b0c948 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 29 Dec 2022 23:29:31 +0000 Subject: [PATCH 06/31] Switch to template for README --- README.md | 50 +++++++----- docs/README.template.md | 171 ++++++++-------------------------------- 2 files changed, 64 insertions(+), 157 deletions(-) diff --git a/README.md b/README.md index 0c25691..824274c 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ This file is auto-generated by a github hook, modify docs/README.template.md ins **RogerThat** is a standalone python web server designed to receive **TradingView** alerts (or similar) and forward them to **Hummingbot**. -**TradingView** is a widely used market market tracker that allows users to create, share and use custom strategies but is quite limited in how it can be "plugged in" to other services. It does however allow the creation of "**Alerts**" which can send data to a webhook (public URL). +**TradingView** is a widely used market market tracker that allows users to create, share and use custom strategies but is quite limited in how it can be "plugged in" or connected to other services. It does however allow the creation of "**Alerts**" which can send data to a Webhook (requires a public-facing URL). **RogerThat** facilitates the collection and forwarding of these **TradingView Alerts** to **Hummingbot** via the **MQTT Bridge** module, which listens to **RogerThat** via subscribed MQTT topics. -Whilst it's purpose is to bridge **TradingView** and **Hummingbot**, it can work as a gateway / bridge between any service that sends data via Webhooks (to a public URL) and serve / route them to *multiple* **Hummingbot** instances. +Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it can work as a gateway / bridge between any service that sends data via Webhooks (to a public URL) and serve / route them to *multiple* **Hummingbot** instances or connected MQTT clients. ##### Menu @@ -41,19 +41,19 @@ Whilst it's purpose is to bridge **TradingView** and **Hummingbot**, it can work ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **When running on Linux be sure to apply the post install steps or you WILL run into permission errors.** -Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip). +Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip).
Linux/Mac ```bash -wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip -unzip master.zip +wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip +unzip mqtt.zip ``` Change directory: ```bash -cd RogerThat-master +cd RogerThat-mqtt ``` ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **You must always run scripts from the main RogerThat directory, do not switch to the `scripts` directory** @@ -64,7 +64,7 @@ cd RogerThat-master
Windows -Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip). +Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). Open up Windows CMD and **switch directory to the extracted zip folder**. @@ -317,8 +317,8 @@ Simple Start command ```json { - "name": "hummingbot_instance_1", - "command": "start", + "topic": "hbot/hummingbot_instance_1/start", + "log_level": "DEBUG" } ``` @@ -326,24 +326,28 @@ Simple Stop command ```json { - "name": "hummingbot_instance_1", - "command": "stop", + "topic": "hbot/hummingbot_instance_1/stop", + "skip_order_cancellation": false } ``` -Alert with all fields using Pine variables +Advanced alert with all fields using Pine variables ```json { - "name": "hummingbot_instance_1", + "topic": "hbot/hummingbot_instance_1/external/events/my_event", + "type": "external_event", "timestamp": "{{timenow}}", - "exchange": "{{exchange}}", - "symbol": "{{ticker}}", - "interval": "{{interval}}", - "price": "{{close}}", - "volume": "{{volume}}", - "command": "{{strategy.market_position}}", - "inventory": "{{strategy.order.comment}}" + "sequence": "{{timenow}}", + "data": { + "exchange": "{{exchange}}", + "symbol": "{{ticker}}", + "interval": "{{interval}}", + "price": "{{close}}", + "volume": "{{volume}}", + "position": "{{strategy.market_position}}", + "inventory": "{{strategy.order.comment}}" + } } ``` @@ -355,7 +359,7 @@ ___ Connect the **Hummingbot** _MQTT Bridge_ and **RogerThat** to the same MQTT Broker. -### Example Config (NEW) +### Example Config
Example Config ... @@ -377,6 +381,8 @@ mqtt_bridge: ### Command Shortcuts +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **This isn't yet implemented.** +
Expand ... @@ -393,6 +399,8 @@ ___ To test basic connection, use any MQTT client and connect to the same broker as RogerThat, then subscribe to the `rogerthat/#` topic. +There is also a small python script in the `examples/` folder which can be used to mimic a TradingView alert. You'll then see the MQTT message if you subscribe to your chosen topic. +
___ diff --git a/docs/README.template.md b/docs/README.template.md index caf4fae..1de6927 100644 --- a/docs/README.template.md +++ b/docs/README.template.md @@ -1,15 +1,15 @@ # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! -## This is an outdated version and no longer supported! See the `mqtt` branch for the latest version. +## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge # RogerThat **RogerThat** is a standalone python web server designed to receive **TradingView** alerts (or similar) and forward them to **Hummingbot**. -**TradingView** is a widely used market market tracker that allows users to create, share and use custom strategies but is quite limited in how it can be "plugged in" to other services. It does however allow the creation of "**Alerts**" which can send data to a webhook (public URL). +**TradingView** is a widely used market market tracker that allows users to create, share and use custom strategies but is quite limited in how it can be "plugged in" or connected to other services. It does however allow the creation of "**Alerts**" which can send data to a Webhook (requires a public-facing URL). -**RogerThat** facilitates the collection and forwarding of these **TradingView Alerts** to **Hummingbot** via the **Remote Command Executor** module, which listens to **RogerThat** via a websocket for received commands and updates. +**RogerThat** facilitates the collection and forwarding of these **TradingView Alerts** to **Hummingbot** via the **MQTT Bridge** module, which listens to **RogerThat** via subscribed MQTT topics. -Whilst it's purpose is to bridge **TradingView** and **Hummingbot**, it can work as a gateway / bridge between any service that sends data via Webhooks (to a public URL) and serve / route them to *multiple* **Hummingbot** instances. +Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it can work as a gateway / bridge between any service that sends data via Webhooks (to a public URL) and serve / route them to *multiple* **Hummingbot** instances or connected MQTT clients. ##### Menu @@ -303,12 +303,9 @@ Where `` is your domain name or public IP address and ### JSON Data for TradingView alerts. -Alerts must be formatted as JSON, the only required parameter is `name`. -*(this parameter key name is configurable in `configs/tradingview.yml`)* +Alerts must be formatted as JSON, the only required parameter is `topic`. -The `name` key or one of the `tradingview_descriptor_fields` must be present in the JSON data to be accepted, but the value can be null or empty. This is transmitted to **Hummingbot** as `event_descriptor` - -Using an empty value for `name` ignores **Hummingbot**'s *Remote Command Executor* routing names. +The `topic` key must be present in the JSON data to be accepted.
Example alert data: @@ -317,8 +314,8 @@ Simple Start command ```json { - "name": "hummingbot_instance_1", - "command": "start", + "topic": "hbot/hummingbot_instance_1/start", + "log_level": "DEBUG" } ``` @@ -326,24 +323,28 @@ Simple Stop command ```json { - "name": "hummingbot_instance_1", - "command": "stop", + "topic": "hbot/hummingbot_instance_1/stop", + "skip_order_cancellation": false } ``` -Alert with all fields using Pine variables +Advanced alert with all fields using Pine variables ```json { - "name": "hummingbot_instance_1", + "topic": "hbot/hummingbot_instance_1/external/events/my_event", + "type": "external_event", "timestamp": "{{timenow}}", - "exchange": "{{exchange}}", - "symbol": "{{ticker}}", - "interval": "{{interval}}", - "price": "{{close}}", - "volume": "{{volume}}", - "command": "{{strategy.market_position}}", - "inventory": "{{strategy.order.comment}}" + "sequence": "{{timenow}}", + "data": { + "exchange": "{{exchange}}", + "symbol": "{{ticker}}", + "interval": "{{interval}}", + "price": "{{close}}", + "volume": "{{volume}}", + "position": "{{strategy.market_position}}", + "inventory": "{{strategy.order.comment}}" + } } ``` @@ -353,95 +354,32 @@ ___ ## Hummingbot Connection -You can connect the **Hummingbot** _Remote Command Executor_ to the **RogerThat** websocket with this URL: -```html -ws://localhost:10073/wss -``` +Connect the **Hummingbot** _MQTT Bridge_ and **RogerThat** to the same MQTT Broker. -### Example Config (NEW) +### Example Config
-Example Config (NEW) ... +Example Config ... -Use something like the following config to connect **RogerThat** to **Hummingbot** via the **Remote Command Executor**. +Use something like the following config to connect **RogerThat** to **Hummingbot** via the **MQTT Bridge**. This config is found inside your main hummingbot folder then `conf\conf_client.yml` ```yaml # Remote commands -remote_command_executor_mode: - remote_command_executor_api_key: a9ba4b61-6f6d-41cf-85c3-7cfdfcbea0f3 - remote_command_executor_ws_url: ws://localhost:10073/wss - # Specify a routing name (for use with multiple Hummingbot instances) - remote_command_executor_routing_name: - # Recommended to keep this on so no events are missed in the case of a network drop out. - remote_command_executor_ignore_first_event: true - # Whether to disable console command processing for remote command events. - # Best to disable this if using in custom scripts or strategies - remote_command_executor_disable_console_commands: false - # You can specify how to translate received commands to Hummingbot commands here - # eg. - # remote_commands_translate_commands: - # long: start - # short: stop - remote_command_executor_translate_commands: - long: start - short: stop +mqtt_bridge: + mqtt_host: localhost + mqtt_port: 1883 + mqtt_autostart: true ```
-### Example Config (OLD) - -
-Example Config (OLD) ... - -Use something like the following config to connect **RogerThat** to **Hummingbot** via the **Remote Command Executor**. - -This config is found inside your main hummingbot folder then `conf\conf_global.yml` - -```yaml -# Remote commands -remote_commands_enabled: true -remote_commands_api_key: a9ba4b61-6f6d-41cf-85c3-7cfdfcbea0f3 -remote_commands_ws_url: ws://localhost:10073/wss -# Specify a routing name (for use with multiple Hummingbot instances) -remote_commands_routing_name: -# Recommended to keep this on so no events are missed in the case of a network drop out. -remote_commands_ignore_first_event: true -# Whether to disable console command processing for remote command events. -# Best to disable this if using in custom scripts or strategies -remote_commands_disable_console_commands: false -# You can specify how to translate received commands to Hummingbot commands here -# eg. -# remote_commands_translate_commands: -# long: start -# short: stop -remote_commands_translate_commands: - long: start - short: stop -``` - -
- - -### Filtering events - -
-Expand ... - -To filter events received based on the `event_descriptor`, change your websockets URL to: -```html -ws://localhost:10073/wss/ -``` - -Where `event-descriptor` matches the `event_descriptor` or *name* value of the events you wish to receive. - -
- ### Command Shortcuts +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **This isn't yet implemented.** +
Expand ... @@ -456,48 +394,9 @@ ___
Test Connection ... -You can enable and disable websockets authentication in the config with these commands. -(You must disable websockets authentication for the in-browser test to work.) - -
-Linux/Mac - -```bash -scripts/setup_config.sh --enable-websocket-auth -scripts/setup_config.sh --disable-websocket-auth -``` -
-
-Windows - -```bat -scripts\setup_config.bat --enable-websocket-auth -scripts\setup_config.bat --disable-websocket-auth -``` -
- -Test the websocket feed in your browser with this js code: - -```javascript -var ws = new WebSocket('ws://localhost:10073/wss'); -ws.onmessage = function (event) { - console.log(event.data); -}; -``` - -Or run the python test listener (requires source installation steps but can authenticate): - -```bash -python tests/test_websocket.py -``` - -Or test the REST url here: - -```html -http://localhost:10073/api/hbot/?api_key= -``` +To test basic connection, use any MQTT client and connect to the same broker as RogerThat, then subscribe to the `rogerthat/#` topic. -Where `` is the generated api key found in `web_server.yml` under `api_allowed_keys_hbot`. +There is also a small python script in the `examples/` folder which can be used to mimic a TradingView alert. You'll then see the MQTT message if you subscribe to your chosen topic.
From a941f204f4e9dbac98e532fe2ed93aa8a6606b57 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Thu, 29 Dec 2022 23:34:51 +0000 Subject: [PATCH 07/31] Linting fixes --- bin/path_util.py | 2 +- rogerthat/config/loader.py | 1 + rogerthat/config/utils.py | 6 ++++-- rogerthat/extensions/extension_quart_json.py | 3 ++- rogerthat/logging/colours.py | 1 - rogerthat/route_handlers/handlers/main.py | 4 +--- rogerthat/utils/path_import.py | 1 + scripts/conda_env_exporter.py | 3 +-- scripts/path_util.py | 3 ++- 9 files changed, 13 insertions(+), 11 deletions(-) diff --git a/bin/path_util.py b/bin/path_util.py index 94e9eb8..2339dab 100644 --- a/bin/path_util.py +++ b/bin/path_util.py @@ -11,6 +11,6 @@ hummingbot.set_prefix_path(os.getcwd()) else: # Dev environment. - from os.path import dirname, realpath import sys + from os.path import dirname, realpath sys.path.insert(0, dirname(dirname(realpath(__file__)))) diff --git a/rogerthat/config/loader.py b/rogerthat/config/loader.py index 20335fd..9e6c3f0 100644 --- a/rogerthat/config/loader.py +++ b/rogerthat/config/loader.py @@ -1,5 +1,6 @@ # Private config vars import sys + from rogerthat.config.utils import config_utils diff --git a/rogerthat/config/utils.py b/rogerthat/config/utils.py index 07f4586..bae99bb 100644 --- a/rogerthat/config/utils.py +++ b/rogerthat/config/utils.py @@ -1,14 +1,16 @@ import os -import ruamel.yaml import secrets import shutil import uuid + +import ruamel.yaml + from rogerthat.utils.yaml import ( load_yml_from_file, save_yml_to_file, + yml_add_to_list, yml_clear_list, yml_fix_list_comments, - yml_add_to_list, ) diff --git a/rogerthat/extensions/extension_quart_json.py b/rogerthat/extensions/extension_quart_json.py index 8894ef2..e7da327 100644 --- a/rogerthat/extensions/extension_quart_json.py +++ b/rogerthat/extensions/extension_quart_json.py @@ -1,6 +1,7 @@ -from quart import json from decimal import Decimal as Dec +from quart import json + class JSONEncoder(json.JSONEncoder): diff --git a/rogerthat/logging/colours.py b/rogerthat/logging/colours.py index 09f4251..3d38351 100644 --- a/rogerthat/logging/colours.py +++ b/rogerthat/logging/colours.py @@ -1,6 +1,5 @@ import logging - BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) RESET_SEQ = "\033[0m" diff --git a/rogerthat/route_handlers/handlers/main.py b/rogerthat/route_handlers/handlers/main.py index dc3b37d..1d30cea 100644 --- a/rogerthat/route_handlers/handlers/main.py +++ b/rogerthat/route_handlers/handlers/main.py @@ -1,6 +1,4 @@ -from quart import ( - jsonify, -) +from quart import jsonify class route_handlers_main: diff --git a/rogerthat/utils/path_import.py b/rogerthat/utils/path_import.py index c51f64b..12cc2c2 100644 --- a/rogerthat/utils/path_import.py +++ b/rogerthat/utils/path_import.py @@ -1,4 +1,5 @@ import os import sys + sys.path.insert(0, os.path.dirname(os.path.realpath(__file__))) sys.path.insert(0, os.getcwd()) diff --git a/scripts/conda_env_exporter.py b/scripts/conda_env_exporter.py index 02ce236..bc7a5d9 100755 --- a/scripts/conda_env_exporter.py +++ b/scripts/conda_env_exporter.py @@ -8,14 +8,13 @@ sign of progress (as of March 2020). TODO (?): support command-line flags -n and -p """ -import re import os +import re import subprocess from functools import cmp_to_key import ruamel.yaml - yaml = ruamel.yaml.YAML() diff --git a/scripts/path_util.py b/scripts/path_util.py index 12e2aba..895e3cf 100644 --- a/scripts/path_util.py +++ b/scripts/path_util.py @@ -1,3 +1,4 @@ -from os.path import dirname, realpath import sys +from os.path import dirname, realpath + sys.path.insert(0, dirname(dirname(realpath(__file__)))) From c36a106dd56ae7e3f39679ca4a4b5b80abafeb0b Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 30 Dec 2022 03:10:46 +0000 Subject: [PATCH 08/31] Fixes for event formatting --- .../versions/18981fafc674_add_is_raw_msg.py | 28 +++++++++++++++++++ examples/test_tradingview_alerts.py | 13 +++++---- rogerthat/config/config.py | 1 + .../config/samples/gateway_mqtt.sample.yml | 2 ++ .../config/samples/tradingview.sample.yml | 4 +-- rogerthat/db/models/tradingview_event.py | 17 +++++++++-- rogerthat/mqtt/mqtt.py | 6 ++-- 7 files changed, 58 insertions(+), 13 deletions(-) create mode 100644 alembic/versions/18981fafc674_add_is_raw_msg.py diff --git a/alembic/versions/18981fafc674_add_is_raw_msg.py b/alembic/versions/18981fafc674_add_is_raw_msg.py new file mode 100644 index 0000000..3477a9d --- /dev/null +++ b/alembic/versions/18981fafc674_add_is_raw_msg.py @@ -0,0 +1,28 @@ +"""Add is_raw_msg + +Revision ID: 18981fafc674 +Revises: d40b06f2cc82 +Create Date: 2022-12-30 02:47:43.782191+00:00 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '18981fafc674' +down_revision = 'd40b06f2cc82' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('tradingview_events', sa.Column('is_raw_msg', sa.Boolean(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('tradingview_events', 'is_raw_msg') + # ### end Alembic commands ### diff --git a/examples/test_tradingview_alerts.py b/examples/test_tradingview_alerts.py index 926f76f..8e2e6b4 100644 --- a/examples/test_tradingview_alerts.py +++ b/examples/test_tradingview_alerts.py @@ -11,18 +11,20 @@ sys.exit(0) +hummingbot_instance = "hummingbot_instance_1" + test_data = { - "topic": "hbot/hummingbot_instance_1/start", - "log_level": "DEBUG" + "topic": f"hbot/{hummingbot_instance}/stop", + "skip_order_cancellation": False, } test_data_list = [ { - "topic": "hbot/hummingbot_instance_1/start", + "topic": f"hbot/{hummingbot_instance}/start", "log_level": "DEBUG" }, { - "topic": "hbot/hummingbot_instance_1/external/events/my_event", + "topic": f"hbot/{hummingbot_instance}/external/events/my_event", "type": "external_event", "timestamp": 1234567890, "sequence": 1234567890, @@ -34,7 +36,8 @@ "volume": "{{volume}}", "position": "{{strategy.market_position}}", "inventory": "{{strategy.order.comment}}" - } + }, + "is_raw_msg": True, } ] diff --git a/rogerthat/config/config.py b/rogerthat/config/config.py index 46fb4fc..15a6513 100644 --- a/rogerthat/config/config.py +++ b/rogerthat/config/config.py @@ -36,6 +36,7 @@ class Config(no_setters): _mqtt_username = _mqtt_config['mqtt_username'] _mqtt_password = _mqtt_config['mqtt_password'] _mqtt_ssl = _mqtt_config['mqtt_ssl'] + _mqtt_reply_topic = _mqtt_config.get("mqtt_reply_topic") or f"{_app_name}/{_mqtt_instance_name}/messages" # Database _database_protocol = _db_config.get("database_protocol", "postgresql+asyncpg") diff --git a/rogerthat/config/samples/gateway_mqtt.sample.yml b/rogerthat/config/samples/gateway_mqtt.sample.yml index aca99a0..c823371 100644 --- a/rogerthat/config/samples/gateway_mqtt.sample.yml +++ b/rogerthat/config/samples/gateway_mqtt.sample.yml @@ -7,6 +7,8 @@ mqtt_enable: true mqtt_instance_name: some-uid +mqtt_reply_topic: + mqtt_host: localhost mqtt_port: 1883 mqtt_username: user diff --git a/rogerthat/config/samples/tradingview.sample.yml b/rogerthat/config/samples/tradingview.sample.yml index 473ba91..d89d861 100644 --- a/rogerthat/config/samples/tradingview.sample.yml +++ b/rogerthat/config/samples/tradingview.sample.yml @@ -5,8 +5,8 @@ template_version: 0.3 # A list of fields or key names to be included in event broadcasts tradingview_include_extra_fields: - - extra_data +- extra_data # A list of fields or key names to be excluded from event broadcasts tradingview_exclude_fields: - - exclude_this_key +- exclude_this_key diff --git a/rogerthat/db/models/tradingview_event.py b/rogerthat/db/models/tradingview_event.py index b123799..b30d7de 100644 --- a/rogerthat/db/models/tradingview_event.py +++ b/rogerthat/db/models/tradingview_event.py @@ -92,6 +92,7 @@ class tradingview_event(db_model_base, days = Column(Numeric) verbose = Column(Boolean) precision = Column(Integer) + is_raw_msg = Column(Boolean) def __init__(self, from_json=None, @@ -114,6 +115,7 @@ def __init__(self, days=None, verbose=None, precision=None, + is_raw_msg=None, ): self.topic = topic self.timestamp_received = int(time.time() * 1000) @@ -135,6 +137,7 @@ def __init__(self, self.days = days self.verbose = verbose self.precision = precision + self.is_raw_msg = is_raw_msg if Config.get_inst().tradingview_include_extra_fields: for evt_key in Config.get_inst().tradingview_include_extra_fields: setattr(self, evt_key, None) @@ -158,6 +161,7 @@ def __init__(self, self.days = decimal_or_none(from_json.get("days")) self.verbose = bool_or_none(from_json.get("verbose")) self.precision = int_or_none(from_json.get("precision")) + self.is_raw_msg = bool_or_none(from_json.get("is_raw_msg")) if Config.get_inst().tradingview_include_extra_fields: for evt_key in Config.get_inst().tradingview_include_extra_fields: setattr(self, evt_key, from_json.get(evt_key)) @@ -169,8 +173,18 @@ async def process_event(self): @property def to_pydantic(self): + if self.is_raw_msg: + evt_dict = self.to_minimised_dict(mqtt=True) + else: + evt_dict = { + "timestamp": self.timestamp_received, + "header": { + 'reply_to': Config.get_inst().mqtt_reply_topic + }, + "data": self.to_minimised_dict(mqtt=True), + } pydantic_model = create_model("TradingviewMessage", - **self.to_minimised_dict(mqtt=True), + **evt_dict, __base__=TradingviewMessage) return pydantic_model() @@ -182,7 +196,6 @@ def to_minimised_dict(self, mqtt=False): val = getattr(self, key) if val is not None: minimised_dict[filtered_event_keys.translate(key)] = val - return minimised_dict @classmethod diff --git a/rogerthat/mqtt/mqtt.py b/rogerthat/mqtt/mqtt.py index 2618be5..e426461 100644 --- a/rogerthat/mqtt/mqtt.py +++ b/rogerthat/mqtt/mqtt.py @@ -43,11 +43,9 @@ def __init__(self, *args, **kwargs): self.mqtt_publisher = None - self.HEARTBEAT_URI = self.HEARTBEAT_URI.replace('$UID', Config.get_inst().mqtt_instance_name) - self.HEARTBEAT_URI = self.HEARTBEAT_URI.replace('$APP', Config.get_inst().app_name) + self.HEARTBEAT_URI = f"{Config.get_inst().app_name}/{Config.get_inst().mqtt_instance_name}/hb" - self.NODE_NAME = self.NODE_NAME.replace('$UID', Config.get_inst().mqtt_instance_name) - self.NODE_NAME = self.NODE_NAME.replace('$APP', Config.get_inst().app_name) + self.NODE_NAME = f"{Config.get_inst().app_name}.{Config.get_inst().mqtt_instance_name}" self._params = self._create_mqtt_params_from_conf() From b11d579bf73addc3cc14782c09046f5544a3e3fb Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 31 Dec 2022 15:36:12 +0000 Subject: [PATCH 09/31] Fix for unpacking json lists in `params` and `data` --- examples/test_tradingview_alerts.py | 10 ++++++++-- rogerthat/db/models/tradingview_event.py | 21 +++++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/examples/test_tradingview_alerts.py b/examples/test_tradingview_alerts.py index 8e2e6b4..49e33ee 100644 --- a/examples/test_tradingview_alerts.py +++ b/examples/test_tradingview_alerts.py @@ -14,8 +14,10 @@ hummingbot_instance = "hummingbot_instance_1" test_data = { - "topic": f"hbot/{hummingbot_instance}/stop", - "skip_order_cancellation": False, + "topic": f"hbot/{hummingbot_instance}/command_shortcuts", + "params": [ + ["spreads", "4", "4"] + ], } test_data_list = [ @@ -38,6 +40,10 @@ "inventory": "{{strategy.order.comment}}" }, "is_raw_msg": True, + }, + { + "topic": f"hbot/{hummingbot_instance}/stop", + "skip_order_cancellation": False, } ] diff --git a/rogerthat/db/models/tradingview_event.py b/rogerthat/db/models/tradingview_event.py index b30d7de..e41b8f6 100644 --- a/rogerthat/db/models/tradingview_event.py +++ b/rogerthat/db/models/tradingview_event.py @@ -41,6 +41,11 @@ "precision", ] +DICT_KEYS = [ + "params", + "data", +] + class filtered_event_keys: _filtered = None @@ -146,7 +151,10 @@ def __init__(self, self.timestamp_event = int_or_none(from_json.get("timestamp")) self.sequence = int_or_none(from_json.get("sequence")) self.event_type = from_json.get("type") - self.params = from_json.get("params") + json_params = from_json.get("params") + if json_params and isinstance(json_params, list): + json_params = {"params": json_params} + self.params = json_params self.data = from_json.get("data") self.msg = from_json.get("msg") self.exchange = from_json.get("exchange") @@ -188,6 +196,14 @@ def to_pydantic(self): __base__=TradingviewMessage) return pydantic_model() + def _unpack_list(self, key, val): + dict_val = None + if key in DICT_KEYS and len(val.keys()) == 1: + unpack_dict = val.get(key) + if unpack_dict and isinstance(unpack_dict, list): + dict_val = unpack_dict + return dict_val + def to_minimised_dict(self, mqtt=False): minimised_dict = {} for i, key in enumerate(filtered_event_keys.list()): @@ -195,7 +211,8 @@ def to_minimised_dict(self, mqtt=False): continue val = getattr(self, key) if val is not None: - minimised_dict[filtered_event_keys.translate(key)] = val + dict_val = self._unpack_list(key, val) + minimised_dict[filtered_event_keys.translate(key)] = dict_val or val return minimised_dict @classmethod From 33f7774bb631c0aceee411b2597b9155bec81872 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Mon, 2 Jan 2023 13:52:19 +0000 Subject: [PATCH 10/31] Test for docker compose versions --- README.md | 2 +- docs/README.template.md | 2 +- scripts/dev/start_docker.bat | 4 ++-- scripts/dev/start_docker.sh | 6 ++++-- scripts/find_docker.sh | 28 +++++++++++++++++++++++++++ scripts/generate_cert_letsencrypt.bat | 10 +++++----- scripts/generate_cert_letsencrypt.sh | 12 +++++++----- scripts/generate_cert_self_signed.bat | 2 +- scripts/generate_cert_self_signed.sh | 4 +++- scripts/start_docker.bat | 2 +- scripts/start_docker.sh | 4 +++- 11 files changed, 56 insertions(+), 20 deletions(-) create mode 100755 scripts/find_docker.sh diff --git a/README.md b/README.md index 824274c..5ca4f73 100644 --- a/README.md +++ b/README.md @@ -134,7 +134,7 @@ ___ Press ctrl+c if running interactively, or run the following command to stop RogerThat: ```bash -docker-compose stop +docker compose stop ``` ___ diff --git a/docs/README.template.md b/docs/README.template.md index 1de6927..00f8c65 100644 --- a/docs/README.template.md +++ b/docs/README.template.md @@ -131,7 +131,7 @@ ___ Press ctrl+c if running interactively, or run the following command to stop RogerThat: ```bash -docker-compose stop +docker compose stop ``` ___ diff --git a/scripts/dev/start_docker.bat b/scripts/dev/start_docker.bat index 50fa03e..16e3a78 100755 --- a/scripts/dev/start_docker.bat +++ b/scripts/dev/start_docker.bat @@ -21,12 +21,12 @@ if not "%1" == "" GOTO Help if %dockerprune% == 1 (ECHO Pruning docker. && docker system prune -f) -if %dontbuild% == 1 (ECHO Skipping build.) else (docker image prune -f && docker-compose build) +if %dontbuild% == 1 (ECHO Skipping build.) else (docker image prune -f && docker compose build) REM Run setup script via docker CALL scripts\setup_config.bat -s -if %nostart% == 1 ( ECHO Skipping start.) else (docker-compose up db rogerthat nginx) +if %nostart% == 1 ( ECHO Skipping start.) else (docker compose up db rogerthat nginx) EXIT /B 0 diff --git a/scripts/dev/start_docker.sh b/scripts/dev/start_docker.sh index 1182dca..90339e1 100755 --- a/scripts/dev/start_docker.sh +++ b/scripts/dev/start_docker.sh @@ -3,6 +3,8 @@ set -e cd $(dirname $0)/../.. +DOCKER_BIN=$(scripts/find_docker.sh) + SCRIPT=`basename ${BASH_SOURCE[0]}` NORM=`tput sgr0` @@ -56,7 +58,7 @@ if [ "${dontbuild} " == "1 " ]; then echo "Skipping build." else docker image prune -f - docker-compose build --progress plain + $DOCKER_BIN build --progress plain fi echo "Calling setup script" @@ -67,5 +69,5 @@ scripts/setup_config.sh -s if [ "${nostart} " == "1 " ]; then echo "Skipping start." else - docker-compose up db rogerthat nginx + $DOCKER_BIN up db rogerthat nginx fi diff --git a/scripts/find_docker.sh b/scripts/find_docker.sh new file mode 100755 index 0000000..7a0427b --- /dev/null +++ b/scripts/find_docker.sh @@ -0,0 +1,28 @@ +docker_compose_bin="" +docker_compose_version="0.0" +docker_version="0.0" +docker_compose_cmd=$(docker-compose --version) +docker_cmd=$(docker compose version) +if [ "${docker_compose_cmd} " != " " ]; then + docker_compose_version_str=$(echo "$docker_compose_cmd" | awk '{split($0,a,"version "); print a[2]}' | awk '{split($0,a,","); print a[1]}') + if [ "${docker_compose_version_str} " != " " ]; then + docker_compose_version=$(echo "$docker_compose_version_str" | awk '{split($0,a,"."); print a[1]"."a[2]}') + fi +fi + +if [ "${docker_cmd} " != " " ]; then + docker_version_str=$(echo "$docker_cmd" | awk '{split($0,a,"version v"); print a[2]}') + if [ "${docker_version_str} " != " " ]; then + docker_version=$(echo "$docker_version_str" | awk '{split($0,a,"."); print a[1]"."a[2]}') + fi +fi + + + +if [ $(bc -l <<<"${docker_compose_version} > ${docker_version}") == "1" ]; then + docker_compose_bin=$(which docker-compose) +else + docker_compose_bin="$(which docker) compose" +fi + +echo "$docker_compose_bin" diff --git a/scripts/generate_cert_letsencrypt.bat b/scripts/generate_cert_letsencrypt.bat index 03bab78..06de652 100755 --- a/scripts/generate_cert_letsencrypt.bat +++ b/scripts/generate_cert_letsencrypt.bat @@ -7,21 +7,21 @@ echo ### Begin generating certificate CALL scripts\setup_config.bat -s -docker-compose stop +docker compose stop echo ### Downloading SSL params ... -docker-compose run --rm --entrypoint "/bin/sh /letsencrypt_download_params.sh" certbot +docker compose run --rm --entrypoint "/bin/sh /letsencrypt_download_params.sh" certbot echo ### Starting nginx ... -docker-compose up --force-recreate -d nginx +docker compose up --force-recreate -d nginx echo. echo ### Requesting Let's Encrypt certificate ... -docker-compose run --rm --entrypoint "/bin/sh /letsencrypt_generate.sh" certbot +docker compose run --rm --entrypoint "/bin/sh /letsencrypt_generate.sh" certbot echo ### Stopping nginx ... -docker-compose stop nginx +docker compose stop nginx echo ### Finished generating certificate you can now start RogerThat! diff --git a/scripts/generate_cert_letsencrypt.sh b/scripts/generate_cert_letsencrypt.sh index 769846a..7c2a13a 100755 --- a/scripts/generate_cert_letsencrypt.sh +++ b/scripts/generate_cert_letsencrypt.sh @@ -5,23 +5,25 @@ cd $(dirname $0)/.. echo "### Begin generating certificate" +DOCKER_BIN=$(scripts/find_docker.sh) + scripts/setup_config.sh -s -docker-compose stop +$DOCKER_BIN stop echo "### Downloading SSL params ..." -docker-compose run --rm --entrypoint "/bin/sh /letsencrypt_download_params.sh" certbot +$DOCKER_BIN run --rm --entrypoint "/bin/sh /letsencrypt_download_params.sh" certbot echo "### Starting nginx ..." -docker-compose up --force-recreate -d nginx +$DOCKER_BIN up --force-recreate -d nginx echo echo "### Requesting Let's Encrypt certificate ..." -docker-compose run --rm --entrypoint "/bin/sh /letsencrypt_generate.sh" certbot +$DOCKER_BIN run --rm --entrypoint "/bin/sh /letsencrypt_generate.sh" certbot echo "### Stopping nginx ..." -docker-compose stop nginx +$DOCKER_BIN stop nginx echo "### Finished generating certificate you can now start RogerThat!" diff --git a/scripts/generate_cert_self_signed.bat b/scripts/generate_cert_self_signed.bat index 9479988..31a996c 100755 --- a/scripts/generate_cert_self_signed.bat +++ b/scripts/generate_cert_self_signed.bat @@ -3,4 +3,4 @@ for %%I in ("%~dp0.") do for %%J in ("%%~dpI.") do set ParentFolderName=%%~dpnxJ cd %ParentFolderName% -docker-compose run --rm --entrypoint "/bin/sh /generate_self_signed_cert.sh" certbot +docker compose run --rm --entrypoint "/bin/sh /generate_self_signed_cert.sh" certbot diff --git a/scripts/generate_cert_self_signed.sh b/scripts/generate_cert_self_signed.sh index 2a8266f..fc445bd 100755 --- a/scripts/generate_cert_self_signed.sh +++ b/scripts/generate_cert_self_signed.sh @@ -3,4 +3,6 @@ cd $(dirname $0)/.. -docker-compose run --rm --entrypoint "/bin/sh /generate_self_signed_cert.sh" certbot +DOCKER_BIN=$(scripts/find_docker.sh) + +$DOCKER_BIN run --rm --entrypoint "/bin/sh /generate_self_signed_cert.sh" certbot diff --git a/scripts/start_docker.bat b/scripts/start_docker.bat index 02fb0d9..f072bfc 100755 --- a/scripts/start_docker.bat +++ b/scripts/start_docker.bat @@ -23,7 +23,7 @@ if %dockerprune% == 1 (ECHO Pruning docker. && docker system prune -f) REM Run setup script via docker CALL scripts\setup_config.bat -s -docker-compose up %daemon% db rogerthat nginx %withcertbot% +docker compose up %daemon% db rogerthat nginx %withcertbot% if not "%daemon%" == "" (scripts\setup_config.bat --print-splash) EXIT /B 0 diff --git a/scripts/start_docker.sh b/scripts/start_docker.sh index 9016aef..8f0c07c 100755 --- a/scripts/start_docker.sh +++ b/scripts/start_docker.sh @@ -7,6 +7,8 @@ fi cd $(dirname $0)/.. +DOCKER_BIN=$(scripts/find_docker.sh) + SCRIPT=`basename ${BASH_SOURCE[0]}` NORM=`tput sgr0` @@ -67,7 +69,7 @@ fi # Run setup script via docker scripts/setup_config.sh -s -docker-compose up${s_dm} db rogerthat nginx${s_cb} +$DOCKER_BIN up${s_dm} db rogerthat nginx${s_cb} if [ "${daemon} " == "1 " ]; then scripts/setup_config.sh --print-splash From 20ad4f9873fd78b269a7bc826df19e316b550783 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Sat, 4 Feb 2023 14:04:16 +0000 Subject: [PATCH 11/31] Fix some docker startup issues --- scripts/find_docker.sh | 13 +++++++++---- scripts/setup_config.sh | 1 + 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/find_docker.sh b/scripts/find_docker.sh index 7a0427b..a172c1d 100755 --- a/scripts/find_docker.sh +++ b/scripts/find_docker.sh @@ -6,20 +6,25 @@ docker_cmd=$(docker compose version) if [ "${docker_compose_cmd} " != " " ]; then docker_compose_version_str=$(echo "$docker_compose_cmd" | awk '{split($0,a,"version "); print a[2]}' | awk '{split($0,a,","); print a[1]}') if [ "${docker_compose_version_str} " != " " ]; then - docker_compose_version=$(echo "$docker_compose_version_str" | awk '{split($0,a,"."); print a[1]"."a[2]}') + docker_compose_version=$(echo "$docker_compose_version_str" | awk '{split($0,a,"."); print a[1]"."a[2]}' | tr -d v) fi fi if [ "${docker_cmd} " != " " ]; then - docker_version_str=$(echo "$docker_cmd" | awk '{split($0,a,"version v"); print a[2]}') + docker_version_str=$(echo "$docker_cmd" | awk '{split($0,a,"version "); print a[2]}') if [ "${docker_version_str} " != " " ]; then - docker_version=$(echo "$docker_version_str" | awk '{split($0,a,"."); print a[1]"."a[2]}') + if [[ "${docker_version_str}" != *"is not a docker command"* ]]; then + docker_version=$(echo "$docker_version_str" | awk '{split($0,a,"."); print a[1]"."a[2]}' | tr -d v) + fi fi fi +compare_str="${docker_compose_version} > ${docker_version}" -if [ $(bc -l <<<"${docker_compose_version} > ${docker_version}") == "1" ]; then +compare_ver=$(bc -l <<<"${compare_str}") + +if [ "$compare_ver" == "1" ]; then docker_compose_bin=$(which docker-compose) else docker_compose_bin="$(which docker) compose" diff --git a/scripts/setup_config.sh b/scripts/setup_config.sh index 8543e85..0d45d0f 100755 --- a/scripts/setup_config.sh +++ b/scripts/setup_config.sh @@ -14,6 +14,7 @@ mkdir -p ./data/db mkdir -p ./data/certbot/certs mkdir -p ./data/certbot/www +chown -R $PUID:$PGID ./configs chown -R $PUID:$PGID ./data docker run -it --rm \ From b8a4083c5dc117de8580f4c41e6242254c4a9db8 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 13:23:52 +0100 Subject: [PATCH 12/31] Change MQTT docker image name --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c34f54f..e5ddea0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,7 +43,7 @@ services: - ./data/certbot/www:/var/www/certbot rogerthat: build: . - image: "theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-latest}" + image: "theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-mqtt}" stop_signal: SIGINT extra_hosts: - "host.docker.internal:host-gateway" From b48b03705c45b32bbc405d2f92e908c9f7e67369 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 13:24:18 +0100 Subject: [PATCH 13/31] Fix docker find script --- scripts/find_docker.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/find_docker.sh b/scripts/find_docker.sh index a172c1d..2994032 100755 --- a/scripts/find_docker.sh +++ b/scripts/find_docker.sh @@ -1,7 +1,7 @@ docker_compose_bin="" docker_compose_version="0.0" docker_version="0.0" -docker_compose_cmd=$(docker-compose --version) +docker_compose_cmd=$(docker-compose --version 2>/dev/null) docker_cmd=$(docker compose version) if [ "${docker_compose_cmd} " != " " ]; then docker_compose_version_str=$(echo "$docker_compose_cmd" | awk '{split($0,a,"version "); print a[2]}' | awk '{split($0,a,","); print a[1]}') @@ -22,7 +22,7 @@ fi compare_str="${docker_compose_version} > ${docker_version}" -compare_ver=$(bc -l <<<"${compare_str}") +compare_ver=$(bc -l <<<"${compare_str}" 2>/dev/null) if [ "$compare_ver" == "1" ]; then docker_compose_bin=$(which docker-compose) From 6277932cc24f1d47e13e8ec427208393e6165448 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 13:44:04 +0100 Subject: [PATCH 14/31] Fix MQTT broadcast error --- rogerthat/queues/mqtt_queue.py | 1 + 1 file changed, 1 insertion(+) diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py index 6ead33e..7d8308b 100644 --- a/rogerthat/queues/mqtt_queue.py +++ b/rogerthat/queues/mqtt_queue.py @@ -75,6 +75,7 @@ async def _listen_for_broadcasts(self): def broadcast(self, event): if not self._mqtt_queue: logger.error("Cannot broadcast, MQTT not started!") + return if self._mqtt: logger.debug("Adding event to MQTT queue.") self._mqtt_queue.put_nowait(event) From b8a439de2913971bbdf8bd0da4551b32060da170 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 14:06:15 +0100 Subject: [PATCH 15/31] Fix empty mqtt username/pass --- rogerthat/config/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rogerthat/config/config.py b/rogerthat/config/config.py index 15a6513..7550b78 100644 --- a/rogerthat/config/config.py +++ b/rogerthat/config/config.py @@ -33,8 +33,8 @@ class Config(no_setters): _mqtt_instance_name = _mqtt_config['mqtt_instance_name'] _mqtt_host = _mqtt_config['mqtt_host'] _mqtt_port = _mqtt_config['mqtt_port'] - _mqtt_username = _mqtt_config['mqtt_username'] - _mqtt_password = _mqtt_config['mqtt_password'] + _mqtt_username = _mqtt_config['mqtt_username'] if len(str(_mqtt_config.get('mqtt_username', '')).strip()) else '' + _mqtt_password = _mqtt_config['mqtt_password'] if len(str(_mqtt_config.get('mqtt_password', '')).strip()) else '' _mqtt_ssl = _mqtt_config['mqtt_ssl'] _mqtt_reply_topic = _mqtt_config.get("mqtt_reply_topic") or f"{_app_name}/{_mqtt_instance_name}/messages" From a3964ec2053c7cfed0fef65f24eedaad59596096 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 14:09:54 +0100 Subject: [PATCH 16/31] Fix MQTT connection refused error logging --- rogerthat/queues/mqtt_queue.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py index 7d8308b..f8f6389 100644 --- a/rogerthat/queues/mqtt_queue.py +++ b/rogerthat/queues/mqtt_queue.py @@ -31,8 +31,8 @@ def __init__(self): self._mqtt = MQTTGateway() self._mqtt.run() self.start() - except (ConnectionRefusedError, gaierror, OSError): - logger.error(f"{self._failure_msg} Check host and port!") + except (ConnectionRefusedError, gaierror, OSError) as e: + logger.error(f"{self._failure_msg} Check host and port! - {e}") except SSLEOFError: logger.error(f"{self._failure_msg} Using plain HTTP port with SSL enabled!") except SSLCertVerificationError: From ac145763a7971cdb95161e9c40d6aa349388d616 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 14:36:05 +0100 Subject: [PATCH 17/31] MQTT gateway debugging --- README.md | 2 ++ rogerthat/queues/mqtt_queue.py | 11 +++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5ca4f73..f8d5c56 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,8 @@ ___ ## Configuration +To connect to an MQTT server on the localhost use `172.17.0.1` or `host.docker.internal` as the mqtt host. + You can use the config setup script via docker by running the following commands:
diff --git a/rogerthat/queues/mqtt_queue.py b/rogerthat/queues/mqtt_queue.py index f8f6389..200b5d3 100644 --- a/rogerthat/queues/mqtt_queue.py +++ b/rogerthat/queues/mqtt_queue.py @@ -25,6 +25,7 @@ def __init__(self): self._mqtt_queue = None self._mqtt: MQTTGateway = None self._failure_msg = "Failed to connect to MQTT Broker!" + self._is_ready = False if Config.get_inst().mqtt_enable: try: @@ -32,7 +33,11 @@ def __init__(self): self._mqtt.run() self.start() except (ConnectionRefusedError, gaierror, OSError) as e: - logger.error(f"{self._failure_msg} Check host and port! - {e}") + if self._mqtt is not None: + extra_debug = f" Parameters: {self._mqtt._params}" + else: + extra_debug = "" + logger.error(f"{self._failure_msg} Check host and port! - {e}.{extra_debug}") except SSLEOFError: logger.error(f"{self._failure_msg} Using plain HTTP port with SSL enabled!") except SSLCertVerificationError: @@ -40,7 +45,8 @@ def __init__(self): except Exception as e: logger.error(e) raise e - logger.info("MQTT Gateway is ready.") + if self._is_ready: + logger.info("MQTT Gateway is ready.") def _create_queue(self): self._mqtt_queue = asyncio.Queue() @@ -51,6 +57,7 @@ def start(self): self._mqtt_queue_task = safe_ensure_future( self._listen_for_broadcasts() ) + self._is_ready = True def stop(self): if self._mqtt_queue_task is not None: From 2d89ae6c34338332b8c343e9b7ad141f53e2cfa2 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 14:55:56 +0100 Subject: [PATCH 18/31] Use rogerthat docker network --- docker-compose.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index e5ddea0..4f6b06f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,8 @@ services: env_file: - ./configs/env_postgres.env user: "${PUID:-999}:${PGID:-999}" + networks: + - rogerthat-bridge nginx: image: theholiestroger/nginx-iptables:latest cap_add: @@ -30,6 +32,8 @@ services: ports: - 80:8080 - 443:8443 + networks: + - rogerthat-bridge certbot: image: certbot/certbot entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'" @@ -56,4 +60,10 @@ services: - "10073:10073" depends_on: - db - entrypoint: ["/home/rogerthat/docker_compose_entrypoint.sh"] \ No newline at end of file + entrypoint: ["/home/rogerthat/docker_compose_entrypoint.sh"] + networks: + - rogerthat-bridge + +networks: + rogerthat-bridge: + driver: bridge \ No newline at end of file From ee61bf0e474280e54165b88b1c80d18a43d5fa5e Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 15:21:25 +0100 Subject: [PATCH 19/31] Update README with MQTT config --- README.md | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 77 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index f8d5c56..25978e0 100644 --- a/README.md +++ b/README.md @@ -141,9 +141,84 @@ ___ ## Configuration -To connect to an MQTT server on the localhost use `172.17.0.1` or `host.docker.internal` as the mqtt host. +Edit the configuration files in `./configs` as needed. -You can use the config setup script via docker by running the following commands: +___ + +### MQTT + +If running a MQTT broker locally (not docker) you should be able to use `localhost` as your `mqtt_host`: + +```yaml +... +mqtt_host: localhost +... +``` + +Running a MQTT broker via Docker Desktop (not advised) you should be able to use `host.docker.internal` as your `mqtt_host`: + +```yaml +... +mqtt_host: host.docker.internal +... +``` + +Running a MQTT broker via Docker in a Linux box on the same host (e.g. the default hummingbot EMQX setup) you'll need to add rogerthat to the same docker network. + +To find the name of the docker network run the command: +```bash +docker network ls +``` + +You should see something like: +``` +e872661fddcc hummingbot_broker_emqx-bridge bridge local +``` + +Edit the `docker-compose.yml` file in the root directory. + +Add the emqx network to the network list at the bottom like this: + +```yaml +... +networks: + rogerthat-bridge: + driver: bridge + hummingbot_broker_emqx-bridge: + external: true + +``` + +Add the emqx network to the rogerthat service like this: +```yaml + ... + entrypoint: ["/home/rogerthat/docker_compose_entrypoint.sh"] + networks: + - rogerthat-bridge + - hummingbot_broker_emqx-bridge + ... +``` + +You can then edit your `configs/gateway_mqtt.yml` file and add the service name e.g. `emqx1` as your `mqtt_host`: + +```yaml +... +mqtt_host: emqx1 +... +``` + +The service name will match the name specified in your MQTT broker compose file e.g.: + +```yaml +services: + emqx1: +``` + +___ + +### Special Tasks + +For certain special tasks, you can use the config setup script via docker by running the following commands:
Linux/Mac From 70e6e53a7739747d29a789bab91038f9b5add2f3 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 15:49:00 +0100 Subject: [PATCH 20/31] Cleanup README some more --- README.md | 56 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 48 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 25978e0..b8bf5ab 100644 --- a/README.md +++ b/README.md @@ -147,6 +147,16 @@ ___ ### MQTT +
+Localhost, Non-Docker Broker + + + +
+ +
+Localhost, Non-Docker Broker + If running a MQTT broker locally (not docker) you should be able to use `localhost` as your `mqtt_host`: ```yaml @@ -155,6 +165,11 @@ mqtt_host: localhost ... ``` +
+ +
+Localhost, Docker Desktop based Broker + Running a MQTT broker via Docker Desktop (not advised) you should be able to use `host.docker.internal` as your `mqtt_host`: ```yaml @@ -163,6 +178,11 @@ mqtt_host: host.docker.internal ... ``` +
+ +
+Localhost, Docker Engine based Broker + Running a MQTT broker via Docker in a Linux box on the same host (e.g. the default hummingbot EMQX setup) you'll need to add rogerthat to the same docker network. To find the name of the docker network run the command: @@ -175,6 +195,8 @@ You should see something like: e872661fddcc hummingbot_broker_emqx-bridge bridge local ``` +The network name in this example is `hummingbot_broker_emqx-bridge`. + Edit the `docker-compose.yml` file in the root directory. Add the emqx network to the network list at the bottom like this: @@ -199,21 +221,39 @@ Add the emqx network to the rogerthat service like this: ... ``` -You can then edit your `configs/gateway_mqtt.yml` file and add the service name e.g. `emqx1` as your `mqtt_host`: +Now run the following command to find the hostname of your MQTT broker replacing `hummingbot_broker_emqx-bridge` as needed: +```bash +docker network inspect hummingbot_broker_emqx-bridge +``` + +You will see the hostname of the MQTT broker in the output like this: -```yaml -... -mqtt_host: emqx1 -... +```json + "Containers": + { + "eb1d17a525cb06a863d10f227cdf7edcd713371fe3f699921360b7b23c512c78": + { + "Name": "hummingbot_broker-emqx1-1", + "EndpointID": "8d8c81332a0284e76246bf0bb19d25987e255d9cf9c43cdeed7df9f5ea436cde", + "MacAddress": "02:42:ac:13:00:02", + "IPv4Address": "172.19.0.2/16", + "IPv6Address": "" + } + } ``` -The service name will match the name specified in your MQTT broker compose file e.g.: +Where `hummingbot_broker-emqx1-1` is the hostname of the MQTT broker in this example. + +You can then edit your `configs/gateway_mqtt.yml` file and add the service name e.g. `hummingbot_broker-emqx1-1` as your `mqtt_host`: ```yaml -services: - emqx1: +... +mqtt_host: hummingbot_broker-emqx1-1 +... ``` +
+ ___ ### Special Tasks From 370c52ca1f649cf9872abee37cb93d424a68db3d Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 15:54:02 +0100 Subject: [PATCH 21/31] Fix README --- README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/README.md b/README.md index b8bf5ab..669483f 100644 --- a/README.md +++ b/README.md @@ -150,13 +150,6 @@ ___
Localhost, Non-Docker Broker - - -
- -
-Localhost, Non-Docker Broker - If running a MQTT broker locally (not docker) you should be able to use `localhost` as your `mqtt_host`: ```yaml From 6b93fbd08a2321efc3397a126aa50269937950e1 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 3 May 2023 18:11:40 +0100 Subject: [PATCH 22/31] Add script to automagically setup EMQX hostname --- README.md | 22 +++++++++ rogerthat/config/utils.py | 94 ++++++++++++++++++++++++++++++++++++++- scripts/setup.py | 4 ++ support/environment.yml | 2 +- 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 669483f..d6c20b0 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,26 @@ mqtt_host: host.docker.internal
Localhost, Docker Engine based Broker +Use the setup command as below to automagically find your EMQX hostname and update compose and config files: + +
+Linux/Mac + +```bash +scripts/setup_config.sh --setup-emqx-docker-hostname +``` +
+
+Windows + +```bat +scripts\setup_config.bat --setup-emqx-docker-hostname +``` +
+ +
+Manual Steps if script failes + Running a MQTT broker via Docker in a Linux box on the same host (e.g. the default hummingbot EMQX setup) you'll need to add rogerthat to the same docker network. To find the name of the docker network run the command: @@ -247,6 +267,8 @@ mqtt_host: hummingbot_broker-emqx1-1
+
+ ___ ### Special Tasks diff --git a/rogerthat/config/utils.py b/rogerthat/config/utils.py index bae99bb..9863f2c 100644 --- a/rogerthat/config/utils.py +++ b/rogerthat/config/utils.py @@ -1,6 +1,8 @@ +import json import os import secrets import shutil +import subprocess import uuid import ruamel.yaml @@ -16,7 +18,8 @@ class config_utils: - _config_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "configs") + _root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + _config_dir = os.path.join(_root_dir, "configs") _config_sample_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "samples") # Config file names @@ -218,3 +221,92 @@ def load_config_tv(cls): @classmethod def load_config_web(cls): return cls.load_config(cls._conf_file_web) + + @classmethod + def load_docker_compose(cls): + return load_yml_from_file(os.path.join(cls._root_dir, "docker-compose.yml")) + + @classmethod + def save_docker_compose(cls, data): + return save_yml_to_file(data, os.path.join(cls._root_dir, "docker-compose.yml")) + + @classmethod + def emqx_mqtt_find_network(cls): + found_network = None + try: + output = subprocess.check_output(["docker", "network", "ls"]) + lines = output.decode().replace("\r", "\n").replace("\n\n", "\n").split("\n") + lines = [line.split() for line in lines if len(line)] + for line in lines: + if len(line) >= 2 and "emqx" in line[1]: + found_network = line[1] + break + except Exception: + pass + return found_network + + @classmethod + def emqx_mqtt_inspect_network(cls, network_name): + found_hostname = None + try: + output = subprocess.check_output(["docker", "network", "inspect", network_name]) + output_data = json.loads(output.decode()) + output_data = output_data[0] if isinstance(output_data, list) and len(output_data) else output_data + containers = output_data["Containers"] + if len(containers): + for container_id, container in containers.items(): + if "emqx" in container["Name"]: + found_hostname = container["Name"] + break + except Exception: + pass + return found_hostname + + @classmethod + def emqx_mqtt_edit_compose_file(cls, emqx_network): + try: + changed = False + compose_data = cls.load_docker_compose() + root_networks = compose_data["networks"] + if emqx_network not in root_networks.keys(): + new_networks_root = dict() + for net, net_conf in root_networks.items(): + if "emqx" in net: + continue + new_networks_root[net] = net_conf + new_networks_root[emqx_network] = {"external": True} + compose_data["networks"] = new_networks_root + changed = True + rogerthat_networks = compose_data["services"]["rogerthat"]["networks"] + if emqx_network not in rogerthat_networks: + new_networks_rthat = list() + for net in rogerthat_networks: + if "emqx" in net: + continue + new_networks_rthat.append(net) + new_networks_rthat.append(emqx_network) + compose_data["services"]["rogerthat"]["networks"] = new_networks_rthat + changed = True + return compose_data if changed else None + except Exception: + return None + + @classmethod + def emqx_mqtt_save_hostname(cls, emqx_host): + config = cls.load_config(cls._conf_file_mqtt) + config["mqtt_host"] = emqx_host + cls.save_config(config, cls._conf_file_mqtt) + + @classmethod + def setup_emqx_docker_hostname(cls): + emqx_network = cls.emqx_mqtt_find_network() + if emqx_network is None: + return + emqx_host = cls.emqx_mqtt_inspect_network(emqx_network) + if emqx_host is None: + return + compose_data = cls.emqx_mqtt_edit_compose_file(emqx_network) + if compose_data is None: + return + cls.emqx_mqtt_save_hostname(emqx_host) + cls.save_docker_compose(compose_data) diff --git a/scripts/setup.py b/scripts/setup.py index 842beb0..55da8a4 100755 --- a/scripts/setup.py +++ b/scripts/setup.py @@ -27,6 +27,8 @@ def parse_args(): help="Disable iptables firewall rules to only allow Cloudflare traffic.") parser.add_argument('--print-splash', dest="print_splash", action='store_true', help="Print launch screen.") + parser.add_argument('--setup-emqx-docker-hostname', dest="setup_emqx_docker_hostname", action='store_true', + help="Setup EMQX MQTT Docker Hostname (see README).") return parser.parse_args() @@ -35,6 +37,8 @@ def parse_args(): if args.print_splash: print(splash_msg) quit() + if args.setup_emqx_docker_hostname: + config_utils.setup_emqx_docker_hostname() if args.delete_configs: config_utils.delete_existing_configs() if args.setup_configs: diff --git a/support/environment.yml b/support/environment.yml index 85eaad3..cd98a81 100644 --- a/support/environment.yml +++ b/support/environment.yml @@ -12,6 +12,7 @@ dependencies: - aiohttp==3.8.3 - alembic==1.7.7 - asyncpg==0.25.0 + - commlib-py==0.10.6 - pre-commit==2.15.0 - python-dateutil==2.8.2 - quart==0.17.0 @@ -20,4 +21,3 @@ dependencies: - sqlalchemy==1.4.32 - ujson==4.1.0 - websockets==10.2 - - git+https://github.com/robotics-4-all/commlib-py.git@f4d11c37f4aaaa20600e748fcb09bf3487f057d1 From 96b714639ca9f71c854c656490da1d18ef751210 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 17 May 2023 16:57:28 +0100 Subject: [PATCH 23/31] Fix permissions on startup --- README.md | 2 +- VERSION | 1 + scripts/dev/start_docker.sh | 2 +- scripts/setup_config.sh | 5 +---- scripts/start_docker.bat | 9 +++++++++ scripts/start_docker.sh | 9 +++++++++ 6 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 VERSION diff --git a/README.md b/README.md index d6c20b0..92b6aaf 100644 --- a/README.md +++ b/README.md @@ -194,7 +194,7 @@ scripts\setup_config.bat --setup-emqx-docker-hostname
-Manual Steps if script failes +Manual Steps if script fails Running a MQTT broker via Docker in a Linux box on the same host (e.g. the default hummingbot EMQX setup) you'll need to add rogerthat to the same docker network. diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..10bf840 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2.0.1 \ No newline at end of file diff --git a/scripts/dev/start_docker.sh b/scripts/dev/start_docker.sh index 90339e1..fb18778 100755 --- a/scripts/dev/start_docker.sh +++ b/scripts/dev/start_docker.sh @@ -51,7 +51,7 @@ if [ "${dockerprune} " == "1 " ]; then echo "Pruning docker." docker system prune -f docker rmi theholiestroger/nginx-iptables:latest || true - docker rmi "theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-latest}" || true + docker rmi "theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-mqtt}" || true fi if [ "${dontbuild} " == "1 " ]; then diff --git a/scripts/setup_config.sh b/scripts/setup_config.sh index 0d45d0f..2d68290 100755 --- a/scripts/setup_config.sh +++ b/scripts/setup_config.sh @@ -14,11 +14,8 @@ mkdir -p ./data/db mkdir -p ./data/certbot/certs mkdir -p ./data/certbot/www -chown -R $PUID:$PGID ./configs -chown -R $PUID:$PGID ./data - docker run -it --rm \ --volume "$(pwd)/configs:/configs" \ -"theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-latest}" \ +"theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-mqtt}" \ ./docker_start_setup_script.sh \ "$@" diff --git a/scripts/start_docker.bat b/scripts/start_docker.bat index f072bfc..7e70881 100755 --- a/scripts/start_docker.bat +++ b/scripts/start_docker.bat @@ -20,6 +20,15 @@ if not "%1" == "" GOTO Help if %dockerprune% == 1 (ECHO Pruning docker. && docker system prune -f) +docker run -it --rm ^ +--volume "./configs:/configs" ^ +--volume "./data:/data" ^ +--volume "./logs:/logs" ^ +--entrypoint "/bin/bash" ^ +--user root ^ +"theholiestroger/rogerthat:mqtt" ^ +"-l" "-c" "chown -R rogerthat:rogerthat /configs /logs; chown -R 999:999 /data" + REM Run setup script via docker CALL scripts\setup_config.bat -s diff --git a/scripts/start_docker.sh b/scripts/start_docker.sh index 8f0c07c..eb9b71f 100755 --- a/scripts/start_docker.sh +++ b/scripts/start_docker.sh @@ -66,6 +66,15 @@ if [ "${daemon} " == "1 " ]; then s_dm=" -d" fi +docker run -it --rm \ +--volume "$(pwd)/configs:/configs" \ +--volume "$(pwd)/data:/data" \ +--volume "$(pwd)/logs:/logs" \ +--entrypoint "/bin/bash" \ +--user root \ +"theholiestroger/rogerthat:${ROGERTHAT_IMG_NAME:-mqtt}" \ +"-l" "-c" "chown -R rogerthat:rogerthat /configs /logs; chown -R ${PUID:-999}:${PGID:-999} /data" + # Run setup script via docker scripts/setup_config.sh -s From 980abdb6f90bf8f4af5b3a4826109d4780feed60 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 17 May 2023 16:59:57 +0100 Subject: [PATCH 24/31] Update ujson --- support/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/support/environment.yml b/support/environment.yml index cd98a81..36bd1cc 100644 --- a/support/environment.yml +++ b/support/environment.yml @@ -19,5 +19,5 @@ dependencies: - quart-auth==0.6.0 - ruamel-yaml==0.17.16 - sqlalchemy==1.4.32 - - ujson==4.1.0 + - ujson==5.7.0 - websockets==10.2 From 94ddd0c66e9bb593d5ff2fd9b78841e2cc380ccc Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Wed, 17 May 2023 18:47:23 +0100 Subject: [PATCH 25/31] Fix isort version for CI --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 484966d..25047fe 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,7 +19,7 @@ repos: files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx types: [file] - repo: https://github.com/pycqa/isort - rev: 5.10.1 + rev: 5.11.5 hooks: - id: isort files: "\\.(py)$" From 66f709319c42e9d4d8a7a553ba931c75112ef544 Mon Sep 17 00:00:00 2001 From: HRWT011 <133653357+HRWT011@users.noreply.github.com> Date: Wed, 17 May 2023 22:34:43 +0200 Subject: [PATCH 26/31] Update README.md I have made the following changes to the Readme file: 1. Docker and Docker Compose were outdated and the links no longer worked. Therefore, Docker Desktop was added for GUI systems (works immediately). 2. For non-GUI systems (servers), Docker and Docker Compose are installed via the terminal. The section "Install Docker Terminal" was added. These changes were tested on Mac Intel and Ubuntu 22.04, 20.04, 18.04. 3. A note was added at the beginning of the README indicating that Mac M1 is not yet supported and enumerating the systems on which RogerThat has been tested (Windows will be added later). 4. Under "Install RogerThat: Linux/Mac", an error is highlighted and a solution for the error is provided. 5. I also added an example JSON file under: "Example Alert Data: Adjusting the Bid and Ask Spreads". --- README.md | 410 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 260 insertions(+), 150 deletions(-) diff --git a/README.md b/README.md index 41f46f0..d410266 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,8 @@ This file is auto-generated by a github hook, modify docs/README.template.md instead. --> # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! -## This is an outdated version and no longer supported! See the `mqtt` branch for the latest version. +## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge +## RogerThat currently works on Linux/Mac Intel and Windows. Mac M1 is not currently supported. It has been tested on Mac Intel and Ubuntu 18.04, 20.04, 22.04. # RogerThat @@ -18,6 +19,7 @@ Whilst it's purpose is to bridge **TradingView** and **Hummingbot**, it can work - [Docker](#docker) * [Installation](#installation) + * [Install Docker Terminal](#install-docker-terminal) * [Running](#running) * [Stopping](#stopping) * [Configuration](#configuration) @@ -31,30 +33,96 @@ Whilst it's purpose is to bridge **TradingView** and **Hummingbot**, it can work # Docker ## Installation -[**Install Docker**](https://docs.docker.com/engine/install/) +[**Install Docker Desktop**](https://www.docker.com/products/docker-desktop/) -[**Install Docker Compose**](https://docker-docs.netlify.app/compose/install/#install-compose) - -![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **Be sure to use the latest docker-compose version number when installing.** - -[**Fix Docker Permissions**](https://docs.docker.com/engine/install/linux-postinstall/) - -![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **When running on Linux be sure to apply the post install steps or you WILL run into permission errors.** - -Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip). +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **Docker Desktop only works if you have a graphical user interface (GUI). For servers, use the installation method: `Install Docker Terminal`.** +--- +## Install Docker Terminal +
+ Linux + + First, you need to make sure your system is up-to-date. Do this by running the following commands: + ```bash + sudo apt-get update && sudo apt-get upgrade + ``` + Next, install some necessary packages that Docker needs: + ```bash + sudo apt-get install apt-transport-https ca-certificates curl software-properties-common + ``` + Add the GPG key for Docker's official repository to your system: + ```bash + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/docker-archive-keyring.gpg >/dev/null + ``` + Add Docker's repository to your APT sources: + ```bash + sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + ``` + Update the package database with Docker packages from the newly added repo: + ```bash + sudo apt-get update + ``` + Make sure you're about to install from the Docker repo instead of the default Ubuntu repo: + ```bash + apt-cache policy docker-ce + ``` + You should see an output similar to the following. The Docker repo path is listed as the candidate for installation: + ```bash + docker-ce: + Installed: (none) + Candidate: 5:20.10.6~3-0~ubuntu-focal + Version table: + 5:20.10.6~3-0~ubuntu-focal 500 + 500 https://download.docker.com/linux/ubuntu focal/stable amd64 Packages + ``` + Now you can install Docker: + ```bash + sudo apt-get install docker-ce + ``` + Docker should now be installed, the daemon started, and the process should run on boot. You can verify this by checking the service's status: + ```bash + sudo systemctl status docker + ``` + + Now we're going to install Docker Compose: + ```bash + sudo curl -L "https://github.com/docker/compose/releases/download/v2.18.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + ``` + Make sure the downloaded file is executable: + ```bash + sudo chmod +x /usr/local/bin/docker-compose + ``` + Verify the installation by displaying the version of Docker Compose: + ```bash + docker-compose --version + ``` + Use this command to add the current user to the Docker group: + ```bash + sudo usermod -aG docker $USER + ``` + This command will make the changes effective without having to log out and log back in: + ```bash + newgrp docker + ``` + ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! Always install the latest version of Docker Compose, which is currently 2.18.0.** + +
+ +--- +## Install RogerThat
Linux/Mac ```bash -wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip -unzip master.zip +wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip +unzip mqtt.zip ``` Change directory: ```bash -cd RogerThat-master +cd RogerThat-mqtt ``` +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! If you encounter an error message when you start it for the first time: `Error - Failed to create tables` you can resolve it by following these steps. Press `Ctrl + C` to exit the program, and then start it again. This should help resolve the issue.** ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **You must always run scripts from the main RogerThat directory, do not switch to the `scripts` directory** @@ -64,7 +132,7 @@ cd RogerThat-master
Windows -Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip). +Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). Open up Windows CMD and **switch directory to the extracted zip folder**. @@ -134,14 +202,146 @@ ___ Press ctrl+c if running interactively, or run the following command to stop RogerThat: ```bash -docker-compose stop +docker compose stop ``` ___ ## Configuration -You can use the config setup script via docker by running the following commands: +Edit the configuration files in `./configs` as needed. + +___ + +### MQTT + +
+Localhost, Non-Docker Broker + +If running a MQTT broker locally (not docker) you should be able to use `localhost` as your `mqtt_host`: + +```yaml +... +mqtt_host: localhost +... +``` + +
+ +
+Localhost, Docker Desktop based Broker + +Running a MQTT broker via Docker Desktop (not advised) you should be able to use `host.docker.internal` as your `mqtt_host`: + +```yaml +... +mqtt_host: host.docker.internal +... +``` + +
+ +
+Localhost, Docker Engine based Broker + +Use the setup command as below to automagically find your EMQX hostname and update compose and config files: + +
+Linux/Mac + +```bash +scripts/setup_config.sh --setup-emqx-docker-hostname +``` +
+
+Windows + +```bat +scripts\setup_config.bat --setup-emqx-docker-hostname +``` +
+ +
+Manual Steps if script fails + +Running a MQTT broker via Docker in a Linux box on the same host (e.g. the default hummingbot EMQX setup) you'll need to add rogerthat to the same docker network. + +To find the name of the docker network run the command: +```bash +docker network ls +``` + +You should see something like: +``` +e872661fddcc hummingbot_broker_emqx-bridge bridge local +``` + +The network name in this example is `hummingbot_broker_emqx-bridge`. + +Edit the `docker-compose.yml` file in the root directory. + +Add the emqx network to the network list at the bottom like this: + +```yaml +... +networks: + rogerthat-bridge: + driver: bridge + hummingbot_broker_emqx-bridge: + external: true + +``` + +Add the emqx network to the rogerthat service like this: +```yaml + ... + entrypoint: ["/home/rogerthat/docker_compose_entrypoint.sh"] + networks: + - rogerthat-bridge + - hummingbot_broker_emqx-bridge + ... +``` + +Now run the following command to find the hostname of your MQTT broker replacing `hummingbot_broker_emqx-bridge` as needed: +```bash +docker network inspect hummingbot_broker_emqx-bridge +``` + +You will see the hostname of the MQTT broker in the output like this: + +```json + "Containers": + { + "eb1d17a525cb06a863d10f227cdf7edcd713371fe3f699921360b7b23c512c78": + { + "Name": "hummingbot_broker-emqx1-1", + "EndpointID": "8d8c81332a0284e76246bf0bb19d25987e255d9cf9c43cdeed7df9f5ea436cde", + "MacAddress": "02:42:ac:13:00:02", + "IPv4Address": "172.19.0.2/16", + "IPv6Address": "" + } + } +``` + +Where `hummingbot_broker-emqx1-1` is the hostname of the MQTT broker in this example. + +You can then edit your `configs/gateway_mqtt.yml` file and add the service name e.g. `hummingbot_broker-emqx1-1` as your `mqtt_host`: + +```yaml +... +mqtt_host: hummingbot_broker-emqx1-1 +... +``` + +
+ +
+ +___ + +### Special Tasks + +For certain special tasks, you can use the config setup script via docker by running the following commands:
Linux/Mac @@ -306,22 +506,30 @@ Where `` is your domain name or public IP address and ### JSON Data for TradingView alerts. -Alerts must be formatted as JSON, the only required parameter is `name`. -*(this parameter key name is configurable in `configs/tradingview.yml`)* +Alerts must be formatted as JSON, the only required parameter is `topic`. -The `name` key or one of the `tradingview_descriptor_fields` must be present in the JSON data to be accepted, but the value can be null or empty. This is transmitted to **Hummingbot** as `event_descriptor` - -Using an empty value for `name` ignores **Hummingbot**'s *Remote Command Executor* routing names. +The `topic` key must be present in the JSON data to be accepted.
Example alert data: +Adjusting the Bid and Ask Spreads + +```json +{ + "topic": "hbot/hummingbot_instance_1/command_shortcuts", + "params": [ + ["spreads", "1", "1"] + ] +} +``` + Simple Start command ```json { - "name": "hummingbot_instance_1", - "command": "start", + "topic": "hbot/hummingbot_instance_1/start", + "log_level": "DEBUG" } ``` @@ -329,24 +537,28 @@ Simple Stop command ```json { - "name": "hummingbot_instance_1", - "command": "stop", + "topic": "hbot/hummingbot_instance_1/stop", + "skip_order_cancellation": false } ``` -Alert with all fields using Pine variables +Advanced alert with all fields using Pine variables ```json { - "name": "hummingbot_instance_1", + "topic": "hbot/hummingbot_instance_1/external/events/my_event", + "type": "external_event", "timestamp": "{{timenow}}", - "exchange": "{{exchange}}", - "symbol": "{{ticker}}", - "interval": "{{interval}}", - "price": "{{close}}", - "volume": "{{volume}}", - "command": "{{strategy.market_position}}", - "inventory": "{{strategy.order.comment}}" + "sequence": "{{timenow}}", + "data": { + "exchange": "{{exchange}}", + "symbol": "{{ticker}}", + "interval": "{{interval}}", + "price": "{{close}}", + "volume": "{{volume}}", + "position": "{{strategy.market_position}}", + "inventory": "{{strategy.order.comment}}" + } } ``` @@ -356,95 +568,32 @@ ___ ## Hummingbot Connection -You can connect the **Hummingbot** _Remote Command Executor_ to the **RogerThat** websocket with this URL: -```html -ws://localhost:10073/wss -``` +Connect the **Hummingbot** _MQTT Bridge_ and **RogerThat** to the same MQTT Broker. -### Example Config (NEW) +### Example Config
-Example Config (NEW) ... +Example Config ... -Use something like the following config to connect **RogerThat** to **Hummingbot** via the **Remote Command Executor**. +Use something like the following config to connect **RogerThat** to **Hummingbot** via the **MQTT Bridge**. This config is found inside your main hummingbot folder then `conf\conf_client.yml` ```yaml # Remote commands -remote_command_executor_mode: - remote_command_executor_api_key: a9ba4b61-6f6d-41cf-85c3-7cfdfcbea0f3 - remote_command_executor_ws_url: ws://localhost:10073/wss - # Specify a routing name (for use with multiple Hummingbot instances) - remote_command_executor_routing_name: - # Recommended to keep this on so no events are missed in the case of a network drop out. - remote_command_executor_ignore_first_event: true - # Whether to disable console command processing for remote command events. - # Best to disable this if using in custom scripts or strategies - remote_command_executor_disable_console_commands: false - # You can specify how to translate received commands to Hummingbot commands here - # eg. - # remote_commands_translate_commands: - # long: start - # short: stop - remote_command_executor_translate_commands: - long: start - short: stop +mqtt_bridge: + mqtt_host: localhost + mqtt_port: 1883 + mqtt_autostart: true ```
-### Example Config (OLD) - -
-Example Config (OLD) ... - -Use something like the following config to connect **RogerThat** to **Hummingbot** via the **Remote Command Executor**. - -This config is found inside your main hummingbot folder then `conf\conf_global.yml` - -```yaml -# Remote commands -remote_commands_enabled: true -remote_commands_api_key: a9ba4b61-6f6d-41cf-85c3-7cfdfcbea0f3 -remote_commands_ws_url: ws://localhost:10073/wss -# Specify a routing name (for use with multiple Hummingbot instances) -remote_commands_routing_name: -# Recommended to keep this on so no events are missed in the case of a network drop out. -remote_commands_ignore_first_event: true -# Whether to disable console command processing for remote command events. -# Best to disable this if using in custom scripts or strategies -remote_commands_disable_console_commands: false -# You can specify how to translate received commands to Hummingbot commands here -# eg. -# remote_commands_translate_commands: -# long: start -# short: stop -remote_commands_translate_commands: - long: start - short: stop -``` - -
- - -### Filtering events - -
-Expand ... - -To filter events received based on the `event_descriptor`, change your websockets URL to: -```html -ws://localhost:10073/wss/ -``` - -Where `event-descriptor` matches the `event_descriptor` or *name* value of the events you wish to receive. - -
- ### Command Shortcuts +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **This isn't yet implemented.** +
Expand ... @@ -459,48 +608,9 @@ ___
Test Connection ... -You can enable and disable websockets authentication in the config with these commands. -(You must disable websockets authentication for the in-browser test to work.) - -
-Linux/Mac - -```bash -scripts/setup_config.sh --enable-websocket-auth -scripts/setup_config.sh --disable-websocket-auth -``` -
-
-Windows - -```bat -scripts\setup_config.bat --enable-websocket-auth -scripts\setup_config.bat --disable-websocket-auth -``` -
- -Test the websocket feed in your browser with this js code: - -```javascript -var ws = new WebSocket('ws://localhost:10073/wss'); -ws.onmessage = function (event) { - console.log(event.data); -}; -``` - -Or run the python test listener (requires source installation steps but can authenticate): - -```bash -python tests/test_websocket.py -``` - -Or test the REST url here: - -```html -http://localhost:10073/api/hbot/?api_key= -``` +To test basic connection, use any MQTT client and connect to the same broker as RogerThat, then subscribe to the `rogerthat/#` topic. -Where `` is the generated api key found in `web_server.yml` under `api_allowed_keys_hbot`. +There is also a small python script in the `examples/` folder which can be used to mimic a TradingView alert. You'll then see the MQTT message if you subscribe to your chosen topic.
@@ -564,4 +674,4 @@ bin/start_rogerthat.py ___ # License -[MIT](https://choosealicense.com/licenses/mit/) \ No newline at end of file +[MIT](https://choosealicense.com/licenses/mit/) From c668742e62635efc8e72ef9ff6d6c2c9b62e2543 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 May 2023 09:32:49 +0100 Subject: [PATCH 27/31] Split README, update template. --- DEVELOPMENT.md | 54 ++++++++ README.md | 75 ++--------- docs/README.template.md | 284 +++++++++++++++++++++++++++++++--------- 3 files changed, 289 insertions(+), 124 deletions(-) create mode 100644 DEVELOPMENT.md diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md new file mode 100644 index 0000000..cbd2776 --- /dev/null +++ b/DEVELOPMENT.md @@ -0,0 +1,54 @@ +# Source Installation (ADVANCED!!!) + +
+Steps to install from **source** instead of docker ... + +## Installation + +[Install Miniconda](https://docs.conda.io/en/latest/miniconda.html) (or Anaconda) + +Clone this repository. + +```bash +git clone git@github.com:TheHolyRoger/RogerThat.git +``` + +Change directory: +```bash +cd RogerThat +``` + +Set up and activate the environment with the following command. + +
+Linux/Mac + +```bash +./scripts/update_environment.sh +``` +
+
+Windows + +```bat +scripts\update_environment.bat +``` +
+ +Run the following command to generate the default configs: +```bash +scripts/setup.py -s +``` + +Edit the configs in `./configs` or via the `setup.py` command. + +___ + +## Running + +From source: + +```bash +bin/start_rogerthat.py +``` +
diff --git a/README.md b/README.md index 0165423..6ad4b38 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ + # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! ## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge ## RogerThat currently works on Linux/Mac Intel and Windows. Mac M1 is not currently supported. It has been tested on Mac Intel and Ubuntu 18.04, 20.04, 22.04. @@ -40,11 +43,16 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c --- ## Install Docker Engine +
+ General Steps + [**Install Docker Engine (Headless servers)**](https://docs.docker.com/engine/install/) [**Fix Docker Permissions**](https://docs.docker.com/engine/install/linux-postinstall/) ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **When running on Linux be sure to apply the post install steps or you WILL run into permission errors.** + +
Debian/Ubuntu Steps @@ -106,19 +114,19 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c --- ## Install RogerThat -Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). +Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/update-readme.zip).
Linux/Mac ```bash -wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip -unzip mqtt.zip +wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/update-readme.zip +unzip update-readme.zip ``` Change directory: ```bash -cd RogerThat-mqtt +cd RogerThat-update-readme ``` ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! If you encounter an error message when you start it for the first time: `Error - Failed to create tables` you can resolve it by following these steps. Press `Ctrl + C` to exit the program, and then start it again. This should help resolve the issue.** @@ -130,7 +138,7 @@ cd RogerThat-mqtt
Windows -Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). +Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/update-readme.zip). Open up Windows CMD and **switch directory to the extracted zip folder**. @@ -614,62 +622,5 @@ There is also a small python script in the `examples/` folder which can be used ___ -# Source Installation (ADVANCED!!!) - -
-Steps to install from **source** instead of docker ... - -## Installation - -[Install Miniconda](https://docs.conda.io/en/latest/miniconda.html) (or Anaconda) - -Clone this repository. - -```bash -git clone git@github.com:TheHolyRoger/RogerThat.git -``` - -Change directory: -```bash -cd RogerThat -``` - -Set up and activate the environment with the following command. - -
-Linux/Mac - -```bash -./scripts/update_environment.sh -``` -
-
-Windows - -```bat -scripts\update_environment.bat -``` -
- -Run the following command to generate the default configs: -```bash -scripts/setup.py -s -``` - -Edit the configs in `./configs` or via the `setup.py` command. - -___ - -## Running - -From source: - -```bash -bin/start_rogerthat.py -``` -
- -___ - # License [MIT](https://choosealicense.com/licenses/mit/) diff --git a/docs/README.template.md b/docs/README.template.md index 00f8c65..88433e9 100644 --- a/docs/README.template.md +++ b/docs/README.template.md @@ -1,5 +1,9 @@ + # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! ## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge +## RogerThat currently works on Linux/Mac Intel and Windows. Mac M1 is not currently supported. It has been tested on Mac Intel and Ubuntu 18.04, 20.04, 22.04. # RogerThat @@ -15,6 +19,7 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c - [Docker](#docker) * [Installation](#installation) + * [Install Docker Engine](#install-docker-engine) * [Running](#running) * [Stopping](#stopping) * [Configuration](#configuration) @@ -28,15 +33,83 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c # Docker ## Installation -[**Install Docker**](https://docs.docker.com/engine/install/) +[**Install Docker Desktop**](https://www.docker.com/products/docker-desktop/) -[**Install Docker Compose**](https://docker-docs.netlify.app/compose/install/#install-compose) +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **Docker Desktop only works if you have a graphical user interface (GUI). For headless servers, use the installation method: `Install Docker Engine`.** -![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **Be sure to use the latest docker-compose version number when installing.** +--- +## Install Docker Engine + +
+ General Steps + +[**Install Docker Engine (Headless servers)**](https://docs.docker.com/engine/install/) [**Fix Docker Permissions**](https://docs.docker.com/engine/install/linux-postinstall/) ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **When running on Linux be sure to apply the post install steps or you WILL run into permission errors.** + +
+ +
+ Debian/Ubuntu Steps + + First, you need to make sure your system is up-to-date. Do this by running the following commands: + ```bash + sudo apt-get update && sudo apt-get upgrade + ``` + Next, uninstall any existing versions: + ```bash + sudo apt-get remove docker docker-engine docker.io containerd runc + ``` + Next, install some necessary packages that Docker needs: + ```bash + sudo apt-get install apt-transport-https ca-certificates curl software-properties-common gnupg + ``` + Add the GPG key for Docker's official repository to your system: + ```bash + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + ``` + Add Docker's repository to your APT sources: + ```bash + echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + ``` + Update the package database with Docker packages from the newly added repo: + ```bash + sudo apt-get update + ``` + Now you can install Docker: + ```bash + sudo apt-get install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + ``` + Docker should now be installed, the daemon started, and the process should run on boot. You can verify this by checking the service's status: + ```bash + sudo systemctl status docker + ``` + + Use this command to create the Docker group if it doesn't exist: + ```bash + sudo groupadd docker + ``` + + Use this command to add the current user to the Docker group: + ```bash + sudo usermod -aG docker $USER + ``` + This command will make the changes effective without having to log out and log back in: + ```bash + newgrp docker + ``` + +
+ +--- +## Install RogerThat Download and extract this [**whole repository**](https://github.com/{{repository.name}}/archive/refs/heads/{{current.branch}}.zip). @@ -52,6 +125,7 @@ Change directory: ```bash cd RogerThat-{{current.branch}} ``` +![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! If you encounter an error message when you start it for the first time: `Error - Failed to create tables` you can resolve it by following these steps. Press `Ctrl + C` to exit the program, and then start it again. This should help resolve the issue.** ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) :warning: **You must always run scripts from the main RogerThat directory, do not switch to the `scripts` directory** @@ -138,7 +212,139 @@ ___ ## Configuration -You can use the config setup script via docker by running the following commands: +Edit the configuration files in `./configs` as needed. + +___ + +### MQTT + +
+Localhost, Non-Docker Broker + +If running a MQTT broker locally (not docker) you should be able to use `localhost` as your `mqtt_host`: + +```yaml +... +mqtt_host: localhost +... +``` + +
+ +
+Localhost, Docker Desktop based Broker + +Running a MQTT broker via Docker Desktop (not advised) you should be able to use `host.docker.internal` as your `mqtt_host`: + +```yaml +... +mqtt_host: host.docker.internal +... +``` + +
+ +
+Localhost, Docker Engine based Broker + +Use the setup command as below to automagically find your EMQX hostname and update compose and config files: + +
+Linux/Mac + +```bash +scripts/setup_config.sh --setup-emqx-docker-hostname +``` +
+
+Windows + +```bat +scripts\setup_config.bat --setup-emqx-docker-hostname +``` +
+ +
+Manual Steps if script fails + +Running a MQTT broker via Docker in a Linux box on the same host (e.g. the default hummingbot EMQX setup) you'll need to add rogerthat to the same docker network. + +To find the name of the docker network run the command: +```bash +docker network ls +``` + +You should see something like: +``` +e872661fddcc hummingbot_broker_emqx-bridge bridge local +``` + +The network name in this example is `hummingbot_broker_emqx-bridge`. + +Edit the `docker-compose.yml` file in the root directory. + +Add the emqx network to the network list at the bottom like this: + +```yaml +... +networks: + rogerthat-bridge: + driver: bridge + hummingbot_broker_emqx-bridge: + external: true + +``` + +Add the emqx network to the rogerthat service like this: +```yaml + ... + entrypoint: ["/home/rogerthat/docker_compose_entrypoint.sh"] + networks: + - rogerthat-bridge + - hummingbot_broker_emqx-bridge + ... +``` + +Now run the following command to find the hostname of your MQTT broker replacing `hummingbot_broker_emqx-bridge` as needed: +```bash +docker network inspect hummingbot_broker_emqx-bridge +``` + +You will see the hostname of the MQTT broker in the output like this: + +```json + "Containers": + { + "eb1d17a525cb06a863d10f227cdf7edcd713371fe3f699921360b7b23c512c78": + { + "Name": "hummingbot_broker-emqx1-1", + "EndpointID": "8d8c81332a0284e76246bf0bb19d25987e255d9cf9c43cdeed7df9f5ea436cde", + "MacAddress": "02:42:ac:13:00:02", + "IPv4Address": "172.19.0.2/16", + "IPv6Address": "" + } + } +``` + +Where `hummingbot_broker-emqx1-1` is the hostname of the MQTT broker in this example. + +You can then edit your `configs/gateway_mqtt.yml` file and add the service name e.g. `hummingbot_broker-emqx1-1` as your `mqtt_host`: + +```yaml +... +mqtt_host: hummingbot_broker-emqx1-1 +... +``` + +
+ +
+ +___ + +### Special Tasks + +For certain special tasks, you can use the config setup script via docker by running the following commands:
Linux/Mac @@ -310,6 +516,17 @@ The `topic` key must be present in the JSON data to be accepted.
Example alert data: +Adjusting the Bid and Ask Spreads + +```json +{ + "topic": "hbot/hummingbot_instance_1/command_shortcuts", + "params": [ + ["spreads", "1", "1"] + ] +} +``` + Simple Start command ```json @@ -402,62 +619,5 @@ There is also a small python script in the `examples/` folder which can be used ___ -# Source Installation (ADVANCED!!!) - -
-Steps to install from **source** instead of docker ... - -## Installation - -[Install Miniconda](https://docs.conda.io/en/latest/miniconda.html) (or Anaconda) - -Clone this repository. - -```bash -git clone git@github.com:{{repository.name}}.git -``` - -Change directory: -```bash -cd RogerThat -``` - -Set up and activate the environment with the following command. - -
-Linux/Mac - -```bash -./scripts/update_environment.sh -``` -
-
-Windows - -```bat -scripts\update_environment.bat -``` -
- -Run the following command to generate the default configs: -```bash -scripts/setup.py -s -``` - -Edit the configs in `./configs` or via the `setup.py` command. - -___ - -## Running - -From source: - -```bash -bin/start_rogerthat.py -``` -
- -___ - # License -[MIT](https://choosealicense.com/licenses/mit/) \ No newline at end of file +[MIT](https://choosealicense.com/licenses/mit/) From 257de8859170fee8be67461d4dea648c1b151ec1 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 May 2023 09:35:36 +0100 Subject: [PATCH 28/31] Fix readme template --- README.md | 3 --- docs/README.template.md | 3 --- 2 files changed, 6 deletions(-) diff --git a/README.md b/README.md index 6ad4b38..557a108 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,6 @@ - # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! ## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge ## RogerThat currently works on Linux/Mac Intel and Windows. Mac M1 is not currently supported. It has been tested on Mac Intel and Ubuntu 18.04, 20.04, 22.04. diff --git a/docs/README.template.md b/docs/README.template.md index 88433e9..60ecfb9 100644 --- a/docs/README.template.md +++ b/docs/README.template.md @@ -1,6 +1,3 @@ - # ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) Important!! RogerThat has been rewritten for MQTT! ## Setup has completely changed to support MQTT with Hummingbot's new MQTT Bridge ## RogerThat currently works on Linux/Mac Intel and Windows. Mac M1 is not currently supported. It has been tested on Mac Intel and Ubuntu 18.04, 20.04, 22.04. From 1583eaf287b106437271b1b0e271fcc6b01c63b2 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 May 2023 09:39:21 +0100 Subject: [PATCH 29/31] Bump readme branch --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 557a108..e8221a4 100644 --- a/README.md +++ b/README.md @@ -111,19 +111,19 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c --- ## Install RogerThat -Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/update-readme.zip). +Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip).
Linux/Mac ```bash -wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/update-readme.zip -unzip update-readme.zip +wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip +unzip mqtt.zip ``` Change directory: ```bash -cd RogerThat-update-readme +cd RogerThat-mqtt ``` ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! If you encounter an error message when you start it for the first time: `Error - Failed to create tables` you can resolve it by following these steps. Press `Ctrl + C` to exit the program, and then start it again. This should help resolve the issue.** @@ -135,7 +135,7 @@ cd RogerThat-update-readme
Windows -Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/update-readme.zip). +Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). Open up Windows CMD and **switch directory to the extracted zip folder**. From 4cda2323a6bb93aac68f5fe96304c4593a572889 Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 May 2023 10:31:31 +0100 Subject: [PATCH 30/31] Update readme branch ready for PR merge --- .githooks/replace_by_git_vars.py | 2 ++ README.md | 10 +++++----- docs/README.template.md | 10 +++++----- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.githooks/replace_by_git_vars.py b/.githooks/replace_by_git_vars.py index 903721f..29ff084 100644 --- a/.githooks/replace_by_git_vars.py +++ b/.githooks/replace_by_git_vars.py @@ -44,6 +44,8 @@ "HEAD", ]).strip().decode('utf-8') +git_vars['current.readme_install_branch'] = "master" + git_vars['current.commit'] = subprocess.check_output([ "git", "rev-parse", diff --git a/README.md b/README.md index e8221a4..4ddb1a4 100644 --- a/README.md +++ b/README.md @@ -111,19 +111,19 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c --- ## Install RogerThat -Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). +Download and extract this [**whole repository**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip).
Linux/Mac ```bash -wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip -unzip mqtt.zip +wget https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip +unzip master.zip ``` Change directory: ```bash -cd RogerThat-mqtt +cd RogerThat-master ``` ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! If you encounter an error message when you start it for the first time: `Error - Failed to create tables` you can resolve it by following these steps. Press `Ctrl + C` to exit the program, and then start it again. This should help resolve the issue.** @@ -135,7 +135,7 @@ cd RogerThat-mqtt
Windows -Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/mqtt.zip). +Manually download and extract the [**repository zip file**](https://github.com/TheHolyRoger/RogerThat/archive/refs/heads/master.zip). Open up Windows CMD and **switch directory to the extracted zip folder**. diff --git a/docs/README.template.md b/docs/README.template.md index 60ecfb9..f29ff9f 100644 --- a/docs/README.template.md +++ b/docs/README.template.md @@ -108,19 +108,19 @@ Whilst RogerThat's purpose is to bridge **TradingView** and **Hummingbot**, it c --- ## Install RogerThat -Download and extract this [**whole repository**](https://github.com/{{repository.name}}/archive/refs/heads/{{current.branch}}.zip). +Download and extract this [**whole repository**](https://github.com/{{repository.name}}/archive/refs/heads/{{current.readme_install_branch}}.zip).
Linux/Mac ```bash -wget https://github.com/{{repository.name}}/archive/refs/heads/{{current.branch}}.zip -unzip {{current.branch}}.zip +wget https://github.com/{{repository.name}}/archive/refs/heads/{{current.readme_install_branch}}.zip +unzip {{current.readme_install_branch}}.zip ``` Change directory: ```bash -cd RogerThat-{{current.branch}} +cd RogerThat-{{current.readme_install_branch}} ``` ![#f03c15](https://placehold.co/15x15/f03c15/f03c15.png) **Important! If you encounter an error message when you start it for the first time: `Error - Failed to create tables` you can resolve it by following these steps. Press `Ctrl + C` to exit the program, and then start it again. This should help resolve the issue.** @@ -132,7 +132,7 @@ cd RogerThat-{{current.branch}}
Windows -Manually download and extract the [**repository zip file**](https://github.com/{{repository.name}}/archive/refs/heads/{{current.branch}}.zip). +Manually download and extract the [**repository zip file**](https://github.com/{{repository.name}}/archive/refs/heads/{{current.readme_install_branch}}.zip). Open up Windows CMD and **switch directory to the extracted zip folder**. From 17b597b173ff619b5f3c0fb68fba13ea28b2f6cf Mon Sep 17 00:00:00 2001 From: TheHolyRoger Date: Fri, 19 May 2023 10:41:58 +0100 Subject: [PATCH 31/31] Add version to log output --- rogerthat/app/rogerthat.py | 2 +- rogerthat/config/config.py | 2 ++ rogerthat/utils/version_number.py | 9 +++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 rogerthat/utils/version_number.py diff --git a/rogerthat/app/rogerthat.py b/rogerthat/app/rogerthat.py index d76e2ce..65ae333 100644 --- a/rogerthat/app/rogerthat.py +++ b/rogerthat/app/rogerthat.py @@ -81,7 +81,7 @@ def start_queues(self): self._mqtt_queue = mqtt_queue.get_instance() def start_server(self): - logger.info("RogerThat startup.") + logger.info(f"RogerThat v{Config.get_inst().version} starting.") self.setup_loop() self.start_queues() logger.info("Starting RogerThat Server.") diff --git a/rogerthat/config/config.py b/rogerthat/config/config.py index 7550b78..96822d8 100644 --- a/rogerthat/config/config.py +++ b/rogerthat/config/config.py @@ -3,6 +3,7 @@ from rogerthat.config.loader import config_loader from rogerthat.utils.class_helpers import no_setters +from rogerthat.utils.version_number import get_version_number _loaded_configs = config_loader() _app_config = _loaded_configs.app_config @@ -15,6 +16,7 @@ class Config(no_setters): # Main App _project_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) + _version = get_version_number(_project_root) _app_name = _app_config['app_name'] _debug_mode = _app_config['debug_mode'] diff --git a/rogerthat/utils/version_number.py b/rogerthat/utils/version_number.py new file mode 100644 index 0000000..67b537e --- /dev/null +++ b/rogerthat/utils/version_number.py @@ -0,0 +1,9 @@ +import os + + +def get_version_number(root_folder): + version = None + with open(os.path.join(root_folder, "VERSION"), "r") as fp: + version = fp.readlines(1)[0] + + return version