Merge branch 'firmware_v0p3'

.gitignore

@@ -1,3 +1,6 @@
 *.zip
 
 *.bak
+electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/.vscode/settings.json
+electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/sdkconfig.pico32
+electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/.vscode/settings.json

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/lib/LovyanGFX_SmartGlasses"]
+	path = electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/lib/LovyanGFX_SmartGlasses
+	url = https://github.com/alex1115alex/LovyanGFX_SmartGlasses.git
+	branch = develop
+[submodule "electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/components/lvgl"]
+	path = electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/components/lvgl
+	url = https://github.com/lvgl/lvgl.git
+	branch = release/v8.3

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/.gitignore

@@ -3,3 +3,5 @@
 .vscode/c_cpp_properties.json
 .vscode/launch.json
 .vscode/ipch
+build/*
+components/lvgl/*

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/.vscode/settings.json

@@ -0,0 +1,69 @@
+{
+    "files.associations": {
+        "lvgl.h": "c",
+        "displaymanager.hpp": "c",
+        "string.h": "c",
+        "stdio.h": "c",
+        "string": "c",
+        "stack": "c",
+        "iostream": "c",
+        "displaymanager.h": "c",
+        "test.h": "c",
+        "array": "cpp",
+        "atomic": "cpp",
+        "*.tcc": "cpp",
+        "cctype": "cpp",
+        "chrono": "cpp",
+        "clocale": "cpp",
+        "cmath": "cpp",
+        "codecvt": "cpp",
+        "condition_variable": "cpp",
+        "csignal": "cpp",
+        "cstdarg": "cpp",
+        "cstddef": "cpp",
+        "cstdint": "cpp",
+        "cstdio": "cpp",
+        "cstdlib": "cpp",
+        "cstring": "cpp",
+        "ctime": "cpp",
+        "cwchar": "cpp",
+        "cwctype": "cpp",
+        "deque": "cpp",
+        "list": "cpp",
+        "unordered_map": "cpp",
+        "vector": "cpp",
+        "exception": "cpp",
+        "algorithm": "cpp",
+        "functional": "cpp",
+        "iterator": "cpp",
+        "map": "cpp",
+        "memory": "cpp",
+        "memory_resource": "cpp",
+        "numeric": "cpp",
+        "optional": "cpp",
+        "random": "cpp",
+        "ratio": "cpp",
+        "set": "cpp",
+        "string_view": "cpp",
+        "system_error": "cpp",
+        "tuple": "cpp",
+        "type_traits": "cpp",
+        "utility": "cpp",
+        "fstream": "cpp",
+        "future": "cpp",
+        "initializer_list": "cpp",
+        "iomanip": "cpp",
+        "iosfwd": "cpp",
+        "istream": "cpp",
+        "limits": "cpp",
+        "mutex": "cpp",
+        "new": "cpp",
+        "ostream": "cpp",
+        "sstream": "cpp",
+        "stdexcept": "cpp",
+        "streambuf": "cpp",
+        "thread": "cpp",
+        "cinttypes": "cpp",
+        "typeinfo": "cpp"
+    }
+}

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/MemoryEval.txt

@@ -0,0 +1,55 @@
+(ONLY NVS + DISPLAY CODE)
+
+With Custom : 13.4% RAM, 50.7% flash, 64352 FREE HEAP
+W/o Cus, 38U: 25.3% RAM, 50.9% flash, 53628 FREE HEAP
+
+AFTER CONFIG_SPIRAM_USE_MALLOC
+
+With Custom : ???% RAM, ???% flash, 4240000 FREE HEAP
+W/o Cus, ???
+
+AFTER ENABLING WIFI INIT FUNCTION...
+
+With Custom : 20% RAM, 85.3% flash, 4240000 FREE HEAP
+W/o Cus, ???
+
+AFTER SETTING...
+CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y
+CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y
+With Custom : 20% RAM, 85.3% flash, 4102000 FREE HEAP [DISPLAY CUTS IN AND OUT]
+W/o Cus, ???
+
+CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=n
+CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=y [DISPLAY CUTS IN AND OUT EVEN WORSE THAN BEFORE] 
+
+CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y
+CONFIG_SPIRAM_ALLOW_BSS_SEG_EXTERNAL_MEMORY=n [DISPLAY CUTS IN AND OUT] 
+With Custom : 18% RAM, 84.3% flash, 4102000 free heap
+w/o custom  : 29.9% RAM, 84.5% flash 4102000 free heap
+
+===============================================================================
+
+LOVYAN ONLY TESTS (NO LVGL/WIFI/ETC):
+
+===================
+ESP-IDF highest res achieved 640x420
+ARDUINO highest res achieved (psram must be enabled) 720x480
+===================
+
+/=/=/=/=/ FUCKED DISPLAY /=/=/=/=/
+PIO + SPIRAM ENABLED + cfgPSRAM={1 or 2}
+6.0% RAM / 50.3% FLASH
+
+*****DISPLAYS FINE*****
+PIO + SPIRAM ENABLED + cfgPSRAM=0
+6.0% RAM / 50.3% FLASH
+
+*****DISPLAYS FINE*****
+PIO + SPIRAM disabled + cfgPSRAM=1
+5.5% RAM / 43.5% FLASH
+
+*****DISPLAYS FINE*****
+PIO + SPIRAM disabled + cfgPSRAM=0
+5.5% RAM / 43.5% FLASH
+
+========================================

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/README.md

@@ -0,0 +1,34 @@
+# OSSG Firmware
+
+This firmware serves as as smart glasses thin client for the [Wearable Intelligence System](https://github.com/emexlabs/WearableIntelligenceSystem) (WIS) for smart glasses powered by ESP32. The WIS Android smart phone (ASP) and GNU/Linux Box (GLBOX) servers handle all of the processing, logic, AI, etc., and this firmware sends signals to the WIS and receives UI prompts from the WIS (i.e. a thin client). See the [WIS](https://github.com/emexlabs/WearableIntelligenceSystem) for the ASP + GLBOX, as well as another frontend with the same API but designed for Android-powered smart glasses (ASG).
+
+## Features
+
+* Displays pretty UI over composite video using LovyanGFX + LVGL
+* Streams recoreded audio to an Android device
+* Receives JSON commands from an Android device
+
+## Setup
+
+1. Install VSCODE
+2. Install platformio in VSCODE
+3. a) If your ESP32 has PSRAM, enable PSRAM in menuconfig. Otherwise, disable PSRAM in menuconfig
+```
+platformio run -t menuconfig
+```
+Component config -> ESP32 Specific -> SPIRAM/PSRAM Enable/Disable  
+b) Comment/uncomment PSRAM lines from platformio.ini depending on your ESP32 hardware
+
+## Gotchas
+
+If build goes well and then at the very end you get undefined reference to a task function that doesn't take arguments, make sure you have it take the `void *args` argument.
+
+## TODO
+
+* Search engine result
+* Command success page (success? And what the command did)
+* Screen to stream in new text (live life captions/speech translate)
+
+## History
+
+This project's display solution is heavily inspired by the [Telescope](https://github.com/alex1115alex/Wearables-Telescope), a Google Glass-inspired project designed to be as compact and cheap as possible by taking advantage of the ESP32's composite video output capabilities. By Alex Israelov.

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/faq.md

@@ -0,0 +1,44 @@
+# Info // FaQ // QnA // Common questions // whatever
+
+**WARNING: Pretty much everything here is subject to change**
+
+## What is the current status of the project?
+
+Here is a link to our todo list / roadmap: (to be inserted)
+
+## What is the current hardware setup?
+
+* ESP32-PICO-D4 with 4mb spiram (I use a TinyPico for development)
+* display module with CVBS driver board
+* I2s microphone
+* Android phone running the WearableIntelligenceSystem app
+
+#### Why an ESP32, and not an Android-based SoC (or other applications processor)?
+
+Size, power consumption, availability, cost, etc are all issues that come with application processors, when all we really need from a client (the glasses) is a display, microphone, and wireless communication with the head unit (Android device).
+
+## What display is (currently) being used
+
+We are using the SONY ECX336C + CVBS driver board + (what our supplier calls the) "freeform optics". 
+
+Some suppliers refer to it as the "OL23C" display module.
+
+#### How can I get one?
+
+You can find them on Alibaba, however, ordering this way tends to be pretty expensive. You may have to message the seller for single unit pricing. We're getting them cheaper @ ~$170/singleUnit directly from a supplier, and will consider doing a bulk order (even cheaper) if enough people are interested. 
+
+NOTE: The freeform optics are only compatible with the ECX336C/OL23C and NOT the older ECX336A/OL23A display module. If you intend on buying the display and freeform optics separately (if that's even possible), make sure you buy the correct display. As far as I know, the only other difference is that the ECX336C/OL23C outputs to a max brightness of 3000cd/m2, while the ECX336A/OL23A only outputs to a max brightness of 800cd/m2.
+
+#### How is the display attached to the optics?
+
+The display + freeform optics come glued together.
+
+## I'd love to contribute to the OSSG firmware, but can't dump hundreds on display hardware! What alternatives exist?
+
+* If you still want a display, I'd recommend the SON-FL08 module. These are about $60 and come with a CVBS driver board and a magnifier. You will need to source your own prism (about $10). For an example of it in action (and links to buy), check out my [Telescope](https://github.com/alex1115alex/wearables-telescope) project.
+
+* If you don't care about the display and just want to contribute while spending the least money, get a Composite-To-USB capture card from Amazon/etc for ~$10. More information about using this in the Telescope readme. 
+
+## Is the “v1” release going to be self-built, or will it come pre-assembled?
+
+As an open source project, you will always be able to build your own OSSG unit. Also, Cayden's company is selling prebuilt units, right now beta units, to help fund the project. You can sign up here: https://us9.list-manage.com/survey?u=18bb4d67053a9c3e4de35fe89&id=586a36974c

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/huge_app.csv

@@ -0,0 +1,5 @@
+# Name,   Type, SubType, Offset,  Size, Flags
+nvs,      data, nvs,     0x9000,  0x5000,
+otadata,  data, ota,     0xe000,  0x2000,
+app0,     app,  ota_0,   0x10000, 0x300000,
+spiffs,   data, spiffs,  0x310000,0xF0000,

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/include/message_types.hpp

@@ -0,0 +1,144 @@
+#pragma once
+
+class MessageTypes {
+    public: 
+        //top level
+        const char* MESSAGE_TYPE_LOCAL = "MESSAGE_TYPE_LOCAL";
+        const char* TIMESTAMP = "TIMESTAMP";
+
+        //DATA TYPES
+        const char* POV_IMAGE = "POV_IMAGE";
+        const char* JPG_BYTES_BASE64 = "JPG_BYTES_BASE64";
+
+        //specific message types (after MESSAGE_TYPE_*)
+        //TRANSCRIPTS
+        const char* FINAL_TRANSCRIPT = "FINAL_TRANSCRIPT";
+        const char* INTERMEDIATE_TRANSCRIPT = "INTERMEDIATE_TRANSCRIPT";
+        const char* TRANSCRIPT_TEXT = "TRANSCRIPT_TEXT";
+        const char* TRANSCRIPT_ID = "TRANSCRIPT_ID";
+
+        //VOICE COMMANDS
+        const char* VOICE_COMMAND_RESPONSE = "VOICE_COMMAND_RESPONSE";
+        const char* COMMAND_RESULT = "COMMAND_RESULT";
+        const char* COMMAND_NAME = "COMMAND_NAME";
+        const char* COMMAND_RESPONSE_DISPLAY_STRING = "COMMAND_RESPONSE_DISPLAY_STRING";
+        //voice command event
+        const char* VOICE_COMMAND_STREAM_EVENT = "VOICE_COMMAND_STREAM_EVENT";
+        const char* VOICE_COMMAND_STREAM_EVENT_TYPE = "VOICE_COMMAND_STREAM_EVENT_TYPE";
+        const char* WAKE_WORD_EVENT_TYPE = "WAKE_WORD_EVENT_TYPE";
+        const char* COMMAND_EVENT_TYPE = "COMMAND_EVENT_TYPE";
+        const char* CANCEL_EVENT_TYPE = "CANCEL_EVENT_TYPE";
+        const char* RESOLVE_EVENT_TYPE = "RESOLVE_EVENT_TYPE";
+        const char* TEXT_RESPONSE_EVENT_TYPE = "TEXT_RESPONSE_EVENT_TYPE";
+        const char* COMMAND_ARGS_EVENT_TYPE = "COMMAND_ARGS_EVENT_TYPE";
+        const char* REQUIRED_ARG_EVENT_TYPE = "REQUIRED_ARG_EVENT_TYPE";
+        const char* ARG_NAME = "ARG_NAME";
+        const char* ARG_OPTIONS = "ARG_OPTIONS";
+        const char* INPUT_VOICE_STRING = "INPUT_VOICE_STRING";
+        const char* VOICE_ARG_EXPECT_TYPE = "VOICE_ARG_EXPECT_TYPE";
+        const char* VOICE_ARG_EXPECT_NATURAL_LANGUAGE = "VOICE_ARG_EXPECT_NATURAL_LANGUAGE";
+        const char* VOICE_COMMAND_LIST = "VOICE_COMMAND_LIST";
+        const char* INPUT_WAKE_WORD = "INPUT_WAKE_WORD";
+        const char* INPUT_VOICE_COMMAND_NAME = "INPUT_VOICE_COMMAND_NAME";
+
+        //AUTOCITER/WEARABLE-REFERENCER
+        const char* AUTOCITER_START = "AUTOCITER_START";
+        const char* AUTOCITER_STOP = "AUTOCITER_STOP";
+        const char* AUTOCITER_PHONE_NUMBER = "AUTOCITER_PHONE_NUMBER";
+        const char* AUTOCITER_POTENTIAL_REFERENCES = "AUTOCITER_POTENTIAL_REFERENCES";
+        const char* AUTOCITER_REFERENCE_DATA = "AUTOCITER_REFERENCE_DATA";
+
+        //request user UI to display a list of possible choices to dipslay
+        const char* REFERENCE_SELECT_REQUEST = "REFERENCE_SELECT_REQUEST";
+        const char* REFERENCES = "REFERENCES";
+
+        //FACE/PERSON SIGHTING
+        const char* FACE_SIGHTING_EVENT = "FACE_SIGHTING_EVENT";
+        const char* FACE_NAME = "FACE_NAME";
+
+        //SMS
+        const char* SMS_REQUEST_SEND = "SMS_REQUEST_SEND";
+        const char* SMS_MESSAGE_TEXT = "SMS_MESSAGE_TEXT";
+        const char* SMS_PHONE_NUMBER = "SMS_PHONE_NUMBER";
+
+        //AUDIO
+        //AUDIO
+        const char* AUDIO_CHUNK_ENCRYPTED = "AUDIO_CHUNK_ENCRYPTED";
+        const char* AUDIO_CHUNK_DECRYPTED = "AUDIO_CHUNK_DECRYPTED";
+        const char* AUDIO_DATA = "AUDIO_DATA";
+
+
+        //COMMS
+        const char* PING = "PING";
+
+        //UI
+        const char* UI_UPDATE_ACTION = "UI_UPDATE_ACTION";
+        const char* PHONE_CONNECTION_STATUS = "PHONE_CONNECTION_STATUS";
+
+    //    //command responses to show
+    //    const char* NATURAL_LANGUAGE_QUERY = "NATURAL_LANGUAGE_QUERY";
+    //    const char* TEXT_RESPONSE = "NATURAL_LANGUAGE_QUERY";
+    //    const char* TEXT_QUERY = "TEXT_QUERY";
+    //    const char* VISUAL_SEARCH_RESULT = "VISUAL_SEARCH_RESULT";
+    //    const char* VISUAL_SEARCH_QUERY = "VISUAL_SEARCH_QUERY";
+    //    const char* VISUAL_SEARCH_IMAGE= "VISUAL_SEARCH_IMAGE";
+    //    const char* VISUAL_SEARCH_DATA = "VISUAL_SEARCH_DATA";
+    //    const char* SEARCH_ENGINE_RESULT = "SEARCH_ENGINE_RESULT";
+    //    const char* SEARCH_ENGINE_RESULT_DATA = "SEARCH_ENGINE_RESULT_DATA";
+    //    const char* TRANSLATION_RESULT = "TRANSLATION_RESULT";
+    //    const char* AFFECTIVE_SUMMARY_RESULT = "AFFECTIVE_SUMMARY_RESULT";
+    //    const char* COMMAND_SWITCH_MODE = "COMMAND_SWITCH_MODE";
+
+        //command responses to show
+        //Natural language
+        const char* NATURAL_LANGUAGE_QUERY = "NATURAL_LANGUAGE_QUERY";
+        const char* TEXT_QUERY = "TEXT_QUERY";
+        //visual search
+        const char* VISUAL_SEARCH_RESULT = "VISUAL_SEARCH_RESULT"; //this is the ASG facing term
+        const char* VISUAL_SEARCH_IMAGE= "VISUAL_SEARCH_IMAGE";
+        const char* VISUAL_SEARCH_QUERY = "VISUAL_SEARCH_QUERY"; //this is the glbox facing term
+        const char* VISUAL_SEARCH_DATA = "VISUAL_SEARCH_DATA"; //this is the payload
+        //search engine
+        const char* SEARCH_ENGINE_QUERY = "SEARCH_ENGINE_QUERY";
+        const char* SEARCH_ENGINE_RESULT = "SEARCH_ENGINE_RESULT";
+        const char* SEARCH_ENGINE_RESULT_DATA = "SEARCH_ENGINE_RESULT_DATA";
+        const char* SEARCH_ENGINE_RESULT_DATA_TITLE = "title";
+        const char* SEARCH_ENGINE_RESULT_DATA_BODY = "body";
+        const char* SEARCH_ENGINE_RESULT_DATA_IMAGE = "image";
+
+        //I/O
+        const char* SG_TOUCHPAD_EVENT = "SG_TOUCHPAD_EVENT";
+        const char* SG_TOUCHPAD_KEYCODE = "SG_TOUCHPAD_KEYCODE";
+
+        //translation
+        const char* TRANSLATE_TEXT_QUERY = "TRANSLATE_TEXT_QUERY";
+        const char* TRANSLATE_TEXT_DATA = "TRANSLATE_TEXT_DATA";
+        const char* TRANSLATE_TEXT_RESULT = "TRANSLATE_TEXT_RESULT";
+        const char* TRANSLATE_TEXT_RESULT_DATA = "TRANSLATION_RESULT_DATA";
+
+        //object translation
+        const char* OBJECT_TRANSLATION_RESULT = "OBJECT_TRANSLATION_RESULT";
+        const char* OBJECT_TRANSLATION_RESULT_DATA = "OBJECT_TRANSLATION_RESULT_DATA";
+
+        const char* AFFECTIVE_SUMMARY_RESULT = "AFFECTIVE_SUMMARY_RESULT";
+        const char* COMMAND_SWITCH_MODE = "COMMAND_SWITCH_MODE";
+
+        //control the current mode of the ASG
+        const char* ACTION_SWITCH_MODES = "ACTION_SWITCH_MODES";
+        const char* NEW_MODE = "NEW_MODE";
+        const char* MODE_VISUAL_SEARCH = "MODE_VISUAL_SEARCH";
+        const char* MODE_LIVE_LIFE_CAPTIONS = "MODE_LIVE_LIFE_CAPTIONS";
+        const char* MODE_HOME = "MODE_HOME";
+        const char* MODE_CONVERSATION_MODE = "MODE_CONVERSATION_MODE";
+        const char* MODE_SOCIAL_MODE = "MODE_SOCIAL_MODE";
+        const char* MODE_REFERENCE_GRID = "MODE_REFERENCE_GRID";
+        const char* MODE_WEARABLE_FACE_RECOGNIZER = "MODE_WEARABLE_FACE_RECOGNIZER";
+        const char* MODE_LANGUAGE_TRANSLATE = "MODE_LANGUAGE_TRANSLATE";
+        const char* MODE_OBJECT_TRANSLATE = "MODE_OBJECT_TRANSLATE";
+        const char* MODE_TEXT_LIST = "MODE_TEXT_LIST";
+        const char* MODE_TEXT_BLOCK = "MODE_TEXT_BLOCK";
+        const char* MODE_BLANK = "MODE_BLANK";
+        const char* MODE_SEARCH_ENGINE_RESULT = "MODE_SEARCH_ENGINE_RESULT";
+};
+
+

electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/include/ossg_constants.hpp

@@ -0,0 +1,11 @@
+#include <string>
+#include <stack>
+using std::string;
+#define PROGRAM_LOG_TAG "OSSG_esp32"
+#define DEVICE_NAME "SmartGlasses_OSSG"
+
+//WIFI
+//#define EXAMPLE_ESP_WIFI_SSID      "WearNet2"
+//#define EXAMPLE_ESP_WIFI_PASS      "ciqi8591"
+static char* esp_wifi_ssid = "WearNet2";
+static char* esp_wifi_pass = "ciqi8591";