feat(display): enabled PSRAM and added logic to draw the album cover …

…on the screen

Cargo.toml

@@ -51,7 +51,6 @@ display-interface-spi = "0.4.1"
 models = { path = "models" }
 graphics = { path = "graphics" }
 crossbeam-channel = "0.5.13"
-jpeg-decoder = "0.3.1"
 
 [build-dependencies]
 embuild = "0.31.3"

graphics/Cargo.toml

@@ -13,11 +13,3 @@ embedded-canvas = "0.2.0"
 embedded-layout = "0.2.0"
 ili9341 = "0.5.0"
 display-interface-spi = "0.4.1"
-# embedded-graphics-core = "0.4.0"
-# image = "0.25.1"
-# embedded-canvas = "0.3.1"
-# embedded-layout = "0.4.1"
-# bytes = "1.6.0"
-# embedded-graphics = "0.7.1"
-# display-interface = "0.4.1"
-# ili9341 = "0.5.0"

graphics/src/lib.rs

@@ -1,5 +1,3 @@
-use std::io::Cursor;
-
 use embedded_canvas::CanvasAt;
 use embedded_graphics::{
     draw_target::DrawTarget,
@@ -13,7 +11,6 @@ use embedded_graphics::{
 };
 use embedded_layout::{layout::linear::LinearLayout, prelude::*};
 use ili9341::DisplayError;
-use image::{io::Reader as ImageReader, DynamicImage, RgbImage};
 
 pub fn rgb888_to_rgb565(r: u8, g: u8, b: u8) -> u16 {
     let red = (r >> 3) as u16;
@@ -65,27 +62,13 @@ where
     );
 }
 
-pub fn draw_album_cover<T>(display: &mut T, image_bytes: Vec<u8>)
+pub fn draw_album_cover<T>(display: &mut T, image_bytes: Option<&[u8]>)
 where
     T: DrawTarget<Color = Rgb565>,
 {
     let mut album_canvas = CanvasAt::new(Point::zero(), Size::new(240, 240));
 
-    let img = ImageReader::new(Cursor::new(image_bytes))
-        .with_guessed_format()
-        .expect("Failed to guess image format")
-        .decode()
-        .expect("Failed to decode image");
-
-    let rgb_imag: RgbImage = img.into_rgb8();
-    let resized_image = DynamicImage::ImageRgb8(rgb_imag)
-        .resize(240, 240, image::imageops::FilterType::Nearest)
-        .to_rgb8();
-
-    let raw = resized_image.clone().into_raw();
-    let rgb565: Vec<u8> = convert_vec_rgb888_to_rgb565(&raw);
-
-    let out: ImageRaw<Rgb565> = ImageRaw::new(&rgb565, resized_image.width());
+    let out: ImageRaw<Rgb565> = ImageRaw::new(&image_bytes.unwrap(), 240);
     out.draw(&mut album_canvas).expect("Could not draw image");
 
     draw_canvas(display, album_canvas, Rgb565::BLACK);
@@ -99,7 +82,6 @@ where
     if displayed_title.len() > 23 {
         displayed_title = title[..22].to_string();
         displayed_title.push_str("..")
-        // displayed_title =  &title[..22].to_owned() + "..".to_string();
     }
 
     let mut displayed_artist = artist.clone();

sdkconfig.defaults

@@ -1,12 +1,21 @@
 # Rust often needs a bit of an extra main task stack size compared to C (the default is 3K)
-CONFIG_ESP_MAIN_TASK_STACK_SIZE=8000
-CONFIG_SPIRAM=y
-CONFIG_SPIRAM_MODE_OCT=y
-CONFIG_SPIRAM_SPEED_80M=y
+CONFIG_ESP_MAIN_TASK_STACK_SIZE=12288
 # Use this to set FreeRTOS kernel tick frequency to 1000 Hz (100 Hz by default).
 # This allows to use 1 ms granuality for thread sleeps (10 ms by default).
 #CONFIG_FREERTOS_HZ=1000
 
 # Workaround for https://github.com/espressif/esp-idf/issues/7631
 #CONFIG_MBEDTLS_CERTIFICATE_BUNDLE=n
 #CONFIG_MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL=n
+CONFIG_SPIRAM_MODE_OCT=y
+CONFIG_SPIRAM_SPEED_80M=y
+CONFIG_MBEDTLS_DEBUG=y
+CONFIG_MBEDTLS_DEBUG_LEVEL=4
+CONFIG_SPIRAM=y
+CONFIG_SPIRAM_BOOT_INIT=y
+CONFIG_SPIRAM_MEMTEST=y
+CONFIG_SPIRAM_TRY_ALLOCATE_WIFI_LWIP=y
+CONFIG_SPIRAM_MALLOC_ALWAYSINTERNAL=102400
+CONFIG_SPIRAM_ALLOW_STACK_EXTERNAL_MEMORY=y
+CONFIG_SPIRAM_USE=CONFIG_SPIRAM_USE_MALLOC
+CONFIG_SPIRAM_USE_MALLOC=y

src/main.rs

@@ -22,7 +22,7 @@ use models::CurrentlyPlaying;
 use once_cell::sync::Lazy;
 use std::{
     sync::{Arc, Mutex},
-    thread::{self},
+    thread,
     time::Duration,
 };
 
@@ -33,7 +33,7 @@ enum ButtonStatus {
 }
 
 enum Signal {
-    ChangeSong(Option<CurrentlyPlaying>),
+    ChangeSong(Option<CurrentlyPlaying>, Option<Arc<[u8]>>),
     UpdateProgress(Option<u32>),
 }
 
@@ -48,6 +48,7 @@ fn main() {
 
     let pins = peripherals.pins;
 
+    // Set up buttons to the right GPIO pins
     let mut btn1_status = ButtonStatus::High;
     let mut btn2_status = ButtonStatus::High;
     let mut btn3_status = ButtonStatus::High;
@@ -59,10 +60,10 @@ fn main() {
     btn_pin3.set_pull(esp_idf_hal::gpio::Pull::Up).unwrap();
     let mut btn_lock = false;
 
+    // Display setup
     let sclk = pins.gpio39;
     let mosi = pins.gpio11;
 
-    // let cs = PinDriver::output(pins.gpio17).unwrap();
     let miso = pins.gpio13;
     let dc = PinDriver::output(pins.gpio15).unwrap();
     let rst = PinDriver::output(pins.gpio16).unwrap();
@@ -83,11 +84,14 @@ fn main() {
         spidispplayinterface,
         rst,
         &mut esp_idf_hal::delay::FreeRtos,
-        ili9341::Orientation::Portrait,
+        ili9341::Orientation::PortraitFlipped,
         ili9341::DisplaySize240x320,
     )
     .expect("Failed to initialize LCD ILI9341.");
 
+    const IMAGE_HEIGHT: u32 = 240;
+    const IMAGE_WIDTH: u32 = 240;
+
     graphics::fill_display(&mut display, Rgb565::BLACK);
 
     let mut wifi_driver = BlockingWifi::wrap(
@@ -120,13 +124,16 @@ fn main() {
     let stored_song_url: Arc<Mutex<Option<String>>> = Arc::new(Mutex::new(None));
     let stored_song_position: Arc<Mutex<Option<u32>>> = Arc::new(Mutex::new(None));
 
+    print_memory_info();
+
+    // This requests the server and gets the current spotify playback in a CurrentlyPlaying struct
     let check_playback = thread::Builder::new()
-        .stack_size(64 * 1024)
+        .stack_size(135 * 1024)
         .spawn(move || {
             loop {
                 let httpconnection = EspHttpConnection::new(&HttpConfig {
-                    // use_global_ca_store: true,
-                    crt_bundle_attach: Some(esp_crt_bundle_attach),
+                    use_global_ca_store: true,
+                    crt_bundle_attach: Some(esp_idf_svc::sys::esp_crt_bundle_attach),
                     ..Default::default()
                 })
                 .expect("Could not establish http connection");
@@ -148,6 +155,8 @@ fn main() {
                 let playing_json: Result<Option<CurrentlyPlaying>, serde_json::Error> =
                     serde_json::from_slice(&playing_buf);
 
+                info!("{:?}", playing_json);
+
                 if let Err(_) = playing_json {
                     Delay::new_default().delay_ms(5000);
                     continue;
@@ -156,13 +165,15 @@ fn main() {
                 let mut prev_song_url = stored_song_url.lock().unwrap();
                 let mut prev_song_position = stored_song_position.lock().unwrap();
 
+                // Depending on the previous and current states, it'll decide what to draw to the
+                // display in order to save ram
                 let playing_data = match playing_json {
                     Ok(Some(data)) => data,
                     _ => {
                         if prev_song_url.is_some() || prev_song_position.is_some() {
                             *prev_song_url = None;
                             *prev_song_position = None;
-                            transmitter.send(Signal::ChangeSong(None));
+                            transmitter.send(Signal::ChangeSong(None, None));
                             transmitter.send(Signal::UpdateProgress(None));
                         }
                         Delay::new_default().delay_ms(5000);
@@ -174,8 +185,19 @@ fn main() {
                     || (prev_song_url.is_some()
                         && *prev_song_url != Some(playing_data.track.name.clone()))
                 {
+                    let mut image_bytes = None::<Arc<[u8]>>;
                     *prev_song_url = Some(playing_data.track.name.clone());
-                    transmitter.send(Signal::ChangeSong(Some(playing_data.clone())));
+                    image_bytes = get_image_bytes(
+                        &*API_URL_ROOT,
+                        &mut httpclient,
+                        &playing_data.track.image_url.clone().unwrap(),
+                        IMAGE_HEIGHT,
+                        IMAGE_WIDTH,
+                    );
+                    transmitter.send(Signal::ChangeSong(
+                        Some(playing_data.clone()),
+                        image_bytes.clone(),
+                    ));
                 }
 
                 if prev_song_position.is_none()
@@ -191,8 +213,9 @@ fn main() {
         })
         .unwrap();
 
+    print_memory_info();
     let control_playback_thread = thread::Builder::new()
-        .stack_size(8 * 1024)
+        .stack_size(4 * 1024)
         .spawn(move || loop {
             if btn_pin1.is_high() && btn1_status == ButtonStatus::Low {
                 info!("Button 1 Pressed - Attempting to skip track");
@@ -241,10 +264,17 @@ fn main() {
     loop {
         match receiver.recv() {
             Ok(received_signal) => match received_signal {
-                Signal::ChangeSong(optional_currently_playing) => {
+                Signal::ChangeSong(optional_currently_playing, optional_image_buf) => {
                     match optional_currently_playing {
                         Some(currently_playing) => {
-                            println!("Attempting to draw image ");
+                            if currently_playing.track.image_url.is_some() {
+                                if optional_image_buf.is_some() {
+                                    graphics::draw_album_cover(
+                                        &mut display,
+                                        optional_image_buf.as_deref(),
+                                    );
+                                }
+                            }
                             graphics::draw_title_and_artist(
                                 &mut display,
                                 currently_playing.track.name,
@@ -309,6 +339,61 @@ fn wifi(wifi_driver: &mut BlockingWifi<EspWifi>) {
     );
 }
 
+// Gets the image bytes from the server. I tried to directly call the image url, process the
+// image bytes and resize the image, but it was too computationally expensive, so I offloaded that
+// logic to the server
+fn get_image_bytes(
+    api_url_root: &String,
+    httpclient: &mut Client<EspHttpConnection>,
+    image_url: &String,
+    image_height: u32,
+    image_width: u32,
+) -> Option<Arc<[u8]>> {
+    let formatted_url = std::format!(
+        "{}/get_resized_image?image_url={}&width={}&height={}",
+        api_url_root,
+        image_url,
+        image_width,
+        image_height
+    );
+    let request = match httpclient.get(&formatted_url) {
+        Ok(req) => req,
+        Err(err) => {
+            error!("Could not initialize request to get image: {:?}", err);
+            return None;
+        }
+    };
+
+    let response = request.submit();
+    if let Err(_) = response {
+        error!("could not get response for image");
+        return None;
+    }
+
+    let mut response = response.unwrap();
+    if response.status() != 200 {
+        error!("status code for getting image: {:?}", response.status());
+        return None;
+    }
+
+    let length = response
+        .header("content-length")
+        .unwrap()
+        .parse::<usize>()
+        .unwrap();
+
+    let mut image_bytes = vec![0u8; length];
+
+    let mut read = 0;
+    while read < length {
+        read += response.read(&mut image_bytes[read..]).unwrap();
+    }
+
+    return Some(Arc::from(graphics::convert_vec_rgb888_to_rgb565(
+        &image_bytes,
+    )));
+}
+
 fn toggle_playback(api_url_root: &String, auth_token: &String) -> bool {
     let httpconnection = EspHttpConnection::new(&HttpConfig {
         // use_global_ca_store: true,