From caaf8fc555f8114c37d584d75709ae39bdf2712d Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Sun, 1 Jan 2023 17:13:49 -0600 Subject: [PATCH 1/2] Add live captions for translation + paragraph breaks --- .../ossg_firmware_mcu/src/display/displaymanager.cpp | 9 ++++++++- .../ossg_firmware_mcu/src/display/displaymanager.hpp | 2 +- .../esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp index 31a239d..bf0e347 100644 --- a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp +++ b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp @@ -140,8 +140,13 @@ void displayEnterVoiceCommandStep3(char* command, char* soFar){ lvgl_release(); } +char * newParagraphChar = "\n\n"; +char * previousTitle = ""; char * currentCaption = ""; -void displayLiveCaptions(char* body = ""){ +void displayLiveCaptions(char* title = "", char* body = ""){ + + if(strcmp(title, previousTitle) != 0) currentCaption = ""; //reset captions if new title + if(strcmp(currentCaption, "") != 0) currentCaption = strcat(currentCaption, newParagraphChar); //If not empty, insert new paragraph currentCaption = strcat(currentCaption, body); string currCapStr(currentCaption); @@ -153,12 +158,14 @@ void displayLiveCaptions(char* body = ""){ if(lv_scr_act() == ui_Card_Live_Captions) //TODO: investigate why this only works sporatically { ESP_LOGI(TAG, "UPDATE LIVE CAPTION SCREEN"); + lv_label_set_text(ui_Card_Live_Captions_Title, title); lv_label_set_text(ui_Card_Live_Captions_Content, currentCaption); lv_obj_scroll_to_y(ui_Card_Live_Captions_Content, SHRT_MAX, LV_ANIM_ON); } else { ESP_LOGI(TAG, "NEW LIVE CAPTION SCREEN"); + lv_label_set_text(ui_Card_Live_Captions_Title, title); lv_label_set_text(ui_Card_Live_Captions_Content, currentCaption); lv_scr_load(ui_Card_Live_Captions); lv_obj_scroll_to_y(ui_Card_Live_Captions_Content, SHRT_MAX, LV_ANIM_OFF); diff --git a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.hpp b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.hpp index 0855e33..b6e3c60 100644 --- a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.hpp +++ b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.hpp @@ -13,7 +13,7 @@ extern void displayEnterVoiceCommandStep2(); extern void displayEnterVoiceCommandStep3(char* command, char* soFar); //extern void displayEnterVoiceCommandStep3(string command, string soFar); extern void displaySearchEngineResult(char* title, char* body, char* image = ""); -extern void displayLiveCaptions(char* body = ""); +extern void displayLiveCaptions(char* title = "", char* body = ""); extern void updateClock(); // extern MessageTypes messageTypesList; // extern char* currentMode; \ No newline at end of file diff --git a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp index 9cb379b..65c02fe 100644 --- a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp +++ b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp @@ -103,7 +103,7 @@ void eventDistributor(void *args){ char * body = (*jsonMessageParser).getJsonKey(messageTypesList.TRANSCRIPT_TEXT); ESP_LOGI(TAG, "BODY IS: %s", body); #if ENABLEDISPLAY - displayLiveCaptions(body); + displayLiveCaptions(title, body); #endif } } else if (!strcmp(messageType, messageTypesList.INTERMEDIATE_TRANSCRIPT)){ @@ -130,6 +130,7 @@ void eventDistributor(void *args){ //call display reference card here with title, body, image arguments #if ENABLEDISPLAY displaySearchEngineResult(title, body); + //displayLiveCaptions(title, body); #endif } else if(!strcmp(messageType, messageTypesList.ACTION_SWITCH_MODES)){ From 02223a13efb58373c3256cebfb8c45fe230982c0 Mon Sep 17 00:00:00 2001 From: Alex Israelov Date: Tue, 3 Jan 2023 14:33:31 -0600 Subject: [PATCH 2/2] Hopefully (but not realistically) v1.0 --- .../src/display/displaymanager.cpp | 35 ++++++++++--------- .../ossg_firmware_mcu/src/main.cpp | 6 ++-- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp index bf0e347..1afe6e5 100644 --- a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp +++ b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/display/displaymanager.cpp @@ -86,13 +86,13 @@ void displayStart(){ //Wipe everything from screen void displayClear(){ lvgl_acquire(); - lv_obj_t * currentScreen = lv_scr_act(); lv_scr_load(blankScreen); lcd.setBrightness(0); lvgl_release(); } void screenTimerCallback(void* arg){ + ESP_LOGI(TAG, "SCREEN TIMER CALLBACK TRIGGERED"); displayClear(); power_to_display(false); } @@ -108,11 +108,12 @@ void updateClock(){ // Call this when turning on or updating the screen // (updates the screen shutoff timer) void updateActivity(int timeoutSeconds = 5){ + ESP_LOGI(TAG, "UPDATE ACTIVITY TRIGGERED"); power_to_display(true); //TODO: evaluate putting this here lcd.setBrightness(128); uint64_t timeoutMicroSeconds = timeoutSeconds * 1000000; - esp_timer_stop(screenTimer); //make sure it's not running anymore + if(timeoutSeconds == -1) timeoutMicroSeconds = INT64_MAX; esp_timer_start_once(screenTimer, timeoutMicroSeconds); } @@ -140,33 +141,33 @@ void displayEnterVoiceCommandStep3(char* command, char* soFar){ lvgl_release(); } -char * newParagraphChar = "\n\n"; -char * previousTitle = ""; -char * currentCaption = ""; +string previousTitle = ""; +string currentCaptionString = ""; void displayLiveCaptions(char* title = "", char* body = ""){ - - if(strcmp(title, previousTitle) != 0) currentCaption = ""; //reset captions if new title - if(strcmp(currentCaption, "") != 0) currentCaption = strcat(currentCaption, newParagraphChar); //If not empty, insert new paragraph - - currentCaption = strcat(currentCaption, body); - string currCapStr(currentCaption); - if(currCapStr.length() > MAX_CAPTION_LENGTH) currCapStr = currCapStr.substr(currCapStr.length()/2); - currentCaption = &currCapStr[0]; - - updateActivity(INT_MAX); + ESP_LOGI(TAG, "DISP LIVE CAPTIONS CALLED WITH TITLE: %s, BODY: %s", title, body); + string titleString(title); + string bodyString(body); + if(titleString != previousTitle) currentCaptionString = ""; + if(currentCaptionString.length() > 0) currentCaptionString += "\n\n"; + previousTitle = titleString; + currentCaptionString = currentCaptionString + bodyString; + //ESP_LOGI(TAG, "NEW CAPTION: " + currentCaptionString); + if(currentCaptionString.length() > MAX_CAPTION_LENGTH) currentCaptionString = currentCaptionString.substr(currentCaptionString.length()/2); + + updateActivity(-1); lvgl_acquire(); if(lv_scr_act() == ui_Card_Live_Captions) //TODO: investigate why this only works sporatically { ESP_LOGI(TAG, "UPDATE LIVE CAPTION SCREEN"); lv_label_set_text(ui_Card_Live_Captions_Title, title); - lv_label_set_text(ui_Card_Live_Captions_Content, currentCaption); + lv_label_set_text(ui_Card_Live_Captions_Content, ¤tCaptionString[0]); lv_obj_scroll_to_y(ui_Card_Live_Captions_Content, SHRT_MAX, LV_ANIM_ON); } else { ESP_LOGI(TAG, "NEW LIVE CAPTION SCREEN"); lv_label_set_text(ui_Card_Live_Captions_Title, title); - lv_label_set_text(ui_Card_Live_Captions_Content, currentCaption); + lv_label_set_text(ui_Card_Live_Captions_Content, ¤tCaptionString[0]); lv_scr_load(ui_Card_Live_Captions); lv_obj_scroll_to_y(ui_Card_Live_Captions_Content, SHRT_MAX, LV_ANIM_OFF); } diff --git a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp index 65c02fe..576279b 100644 --- a/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp +++ b/electronics_and_firmware/esp32_firmware_v0p3/ossg_firmware_mcu/src/main.cpp @@ -129,8 +129,8 @@ void eventDistributor(void *args){ char * body = (*jsonMessageParser).getJsonKey(messageTypesList.TRANSLATE_TEXT_RESULT_DATA); //call display reference card here with title, body, image arguments #if ENABLEDISPLAY - displaySearchEngineResult(title, body); - //displayLiveCaptions(title, body); + //displaySearchEngineResult(title, body); + displayLiveCaptions(title, body); #endif } else if(!strcmp(messageType, messageTypesList.ACTION_SWITCH_MODES)){ @@ -194,7 +194,7 @@ void app_main(void) //setup eventsDistributor and eventsBuffer, which handles incoming data from WIS and calls functions based on what WIS tells us to do eventsBuffer = xMessageBufferCreate(eventsBufferLen); TaskHandle_t eventsTask = NULL; - xTaskCreate(eventDistributor, "events_distribution_task", 6*1024, NULL, 1, &eventsTask); + xTaskCreate(eventDistributor, "events_distribution_task", 10*1024, NULL, 1, &eventsTask); //connect to WIS web socket websocketSendBuffer = xMessageBufferCreate(websocketSendBufferLen);