From 30465124c5d2f371650e201094965abbd1cb9852 Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Sat, 30 Nov 2024 15:44:50 +0100 Subject: [PATCH 1/9] Remove Detection constructor dependency on temporary audio file This allows to decorrelate the Detection class and the .wav audio files that are temporarily stored in the "StreamData" folder. --- scripts/server.py | 5 ++--- scripts/utils/helpers.py | 15 +++++++-------- scripts/utils/reporting.py | 27 +++++++++++++++++++-------- 3 files changed, 28 insertions(+), 19 deletions(-) diff --git a/scripts/server.py b/scripts/server.py index 855223491..5b673c61f 100644 --- a/scripts/server.py +++ b/scripts/server.py @@ -339,9 +339,8 @@ def run_analysis(file): log.warning("Excluded as below Species Occurrence Frequency Threshold: %s", entry[0]) else: d = Detection( - file.file_date, - time_slot.split(';')[0], - time_slot.split(';')[1], + file.file_date + datetime.timedelta(seconds=float(time_slot.split(';')[0])), + file.file_date + datetime.timedelta(seconds=float(time_slot.split(';')[1])), entry[0], entry[1], ) diff --git a/scripts/utils/helpers.py b/scripts/utils/helpers.py index 864f0157f..0ef50ff73 100644 --- a/scripts/utils/helpers.py +++ b/scripts/utils/helpers.py @@ -42,14 +42,13 @@ def get_settings(settings_path='/etc/birdnet/birdnet.conf', force_reload=False): class Detection: - def __init__(self, file_date, start_time, stop_time, species, confidence): - self.start = float(start_time) - self.stop = float(stop_time) - self.datetime = file_date + datetime.timedelta(seconds=self.start) - self.date = self.datetime.strftime("%Y-%m-%d") - self.time = self.datetime.strftime("%H:%M:%S") - self.iso8601 = self.datetime.astimezone(get_localzone()).isoformat() - self.week = self.datetime.isocalendar()[1] + def __init__(self, start_datetime, stop_datetime, species, confidence): + self.start_datetime = start_datetime + self.stop_datetime = stop_datetime + self.date = self.start_datetime.strftime("%Y-%m-%d") + self.time = self.start_datetime.strftime("%H:%M:%S") + self.iso8601 = self.start_datetime.astimezone(get_localzone()).isoformat() + self.week = self.start_datetime.isocalendar()[1] self.confidence = round(float(confidence), 4) self.confidence_pct = round(self.confidence * 100) self.species = species diff --git a/scripts/utils/reporting.py b/scripts/utils/reporting.py index ac0e00997..ea8f920c7 100644 --- a/scripts/utils/reporting.py +++ b/scripts/utils/reporting.py @@ -70,7 +70,12 @@ def extract_detection(file: ParseFileName, detection: Detection): log.warning('Extraction exists. Moving on: %s', new_file) else: os.makedirs(new_dir, exist_ok=True) - extract_safe(file.file_name, new_file, detection.start, detection.stop) + extract_safe( + file.file_name, + new_file, + (detection.start_datetime - file.file_date).seconds, + (detection.stop_datetime - file.file_date).seconds, + ) spectrogram(new_file, detection.common_name, new_file.replace(os.path.expanduser('~/'), '')) return new_file @@ -130,7 +135,7 @@ def write_to_json_file(file: ParseFileName, detections: [Detection]): json_file = f'{file.file_name}.json' log.debug(f'WRITING RESULTS TO {json_file}') dets = {'file_name': os.path.basename(json_file), 'timestamp': file.iso8601, 'delay': conf['RECORDING_LENGTH'], - 'detections': [{"start": det.start, "common_name": det.common_name, "confidence": det.confidence} for det in + 'detections': [{"start": (det.start_datetime - file.file_date).seconds, "common_name": det.common_name, "confidence": det.confidence} for det in detections]} with open(json_file, 'w') as rfile: rfile.write(json.dumps(dets)) @@ -184,12 +189,18 @@ def bird_weather(file: ParseFileName, detections: [Detection]): # POST detection to server detection_url = f'https://app.birdweather.com/api/v1/stations/{conf["BIRDWEATHER_ID"]}/detections' - data = {'timestamp': detection.iso8601, 'lat': conf['LATITUDE'], 'lon': conf['LONGITUDE'], - 'soundscapeId': soundscape_id, - 'soundscapeStartTime': detection.start, 'soundscapeEndTime': detection.stop, - 'commonName': detection.common_name, 'scientificName': detection.scientific_name, - 'algorithm': '2p4' if conf['MODEL'] == 'BirdNET_GLOBAL_6K_V2.4_Model_FP16' else 'alpha', - 'confidence': detection.confidence} + data = { + 'timestamp': detection.iso8601, + 'lat': conf['LATITUDE'], + 'lon': conf['LONGITUDE'], + 'soundscapeId': soundscape_id, + 'soundscapeStartTime': (detection.start_datetime - file.file_date).seconds, + 'soundscapeEndTime': (detection.stop_datetime - file.file_date).seconds, + 'commonName': detection.common_name, + 'scientificName': detection.scientific_name, + 'algorithm': '2p4' if conf['MODEL'] == 'BirdNET_GLOBAL_6K_V2.4_Model_FP16' else 'alpha', + 'confidence': detection.confidence, + } log.debug(data) try: From 0afd5ba0c88c1d5c8cc0a1e2d234d8207fa1c677 Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Fri, 6 Dec 2024 09:58:36 +0100 Subject: [PATCH 2/9] Move setup_logging to helpers.py This allows to reuse this function in another service. --- scripts/birdnet_analysis.py | 16 ++-------------- scripts/utils/helpers.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/scripts/birdnet_analysis.py b/scripts/birdnet_analysis.py index ea2a5ceab..df2921b59 100644 --- a/scripts/birdnet_analysis.py +++ b/scripts/birdnet_analysis.py @@ -3,7 +3,6 @@ import os.path import re import signal -import sys import threading from queue import Queue from subprocess import CalledProcessError @@ -12,13 +11,13 @@ from inotify.constants import IN_CLOSE_WRITE from server import load_global_model, run_analysis -from utils.helpers import get_settings, ParseFileName, get_wav_files, ANALYZING_NOW +from utils.helpers import get_settings, ParseFileName, get_wav_files, ANALYZING_NOW, setup_logging from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, bird_weather, heartbeat, \ update_json_file shutdown = False -log = logging.getLogger(__name__) +log = logging.getLogger(os.path.splitext(os.path.basename(os.path.realpath(__file__)))[0]) def sig_handler(sig_num, curr_stack_frame): @@ -129,17 +128,6 @@ def handle_reporting_queue(queue): log.info('handle_reporting_queue done') -def setup_logging(): - logger = logging.getLogger() - formatter = logging.Formatter("[%(name)s][%(levelname)s] %(message)s") - handler = logging.StreamHandler(stream=sys.stdout) - handler.setFormatter(formatter) - logger.addHandler(handler) - logger.setLevel(logging.INFO) - global log - log = logging.getLogger('birdnet_analysis') - - if __name__ == '__main__': signal.signal(signal.SIGINT, sig_handler) signal.signal(signal.SIGTERM, sig_handler) diff --git a/scripts/utils/helpers.py b/scripts/utils/helpers.py index 0ef50ff73..f69bd00b6 100644 --- a/scripts/utils/helpers.py +++ b/scripts/utils/helpers.py @@ -5,6 +5,8 @@ import subprocess from configparser import ConfigParser from itertools import chain +import logging +import sys from tzlocal import get_localzone @@ -101,3 +103,12 @@ def get_wav_files(): open_recs = get_open_files_in_dir(rec_dir) files = [file for file in files if file not in open_recs] return files + + +def setup_logging(): + logger = logging.getLogger() + formatter = logging.Formatter("[%(name)s][%(levelname)s] %(message)s") + handler = logging.StreamHandler(stream=sys.stdout) + handler.setFormatter(formatter) + logger.addHandler(handler) + logger.setLevel(logging.INFO) From eb554f4a866660ac94c80a0ebb6522ea5bc45b3a Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Fri, 6 Dec 2024 10:04:16 +0100 Subject: [PATCH 3/9] Add birdweather.py module --- scripts/birdnet_analysis.py | 6 +- scripts/utils/birdweather.py | 123 +++++++++++++++++++++++++++++++++++ scripts/utils/reporting.py | 65 +++++++----------- 3 files changed, 148 insertions(+), 46 deletions(-) create mode 100644 scripts/utils/birdweather.py diff --git a/scripts/birdnet_analysis.py b/scripts/birdnet_analysis.py index df2921b59..5cdf305bd 100644 --- a/scripts/birdnet_analysis.py +++ b/scripts/birdnet_analysis.py @@ -12,8 +12,8 @@ from server import load_global_model, run_analysis from utils.helpers import get_settings, ParseFileName, get_wav_files, ANALYZING_NOW, setup_logging -from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, bird_weather, heartbeat, \ - update_json_file +from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, \ + post_current_detections_to_birdweather, heartbeat, update_json_file shutdown = False @@ -114,7 +114,7 @@ def handle_reporting_queue(queue): write_to_file(file, detection) write_to_db(file, detection) apprise(file, detections) - bird_weather(file, detections) + post_current_detections_to_birdweather(file, detections) heartbeat() os.remove(file.file_name) except BaseException as e: diff --git a/scripts/utils/birdweather.py b/scripts/utils/birdweather.py new file mode 100644 index 000000000..f843b1ac1 --- /dev/null +++ b/scripts/utils/birdweather.py @@ -0,0 +1,123 @@ +"""Module to handle communication with the BirdWeather API.""" + +import requests +import logging +import datetime + +import gzip +from typing import Any, Dict, List, Optional +from .helpers import Detection + +log = logging.getLogger(__name__) + + +def get_birdweather_species_id(sci_name: str, com_name: str) -> int: + """Lookup a BirdWeather species ID based on the species scientific and common names.""" + species_url = "https://app.birdweather.com/api/v1/species/lookup" + try: + resp = requests.post( + url=species_url, + json={"species": [f"{sci_name}_{com_name}"]}, + timeout=20, + ) + data = resp.json() + if not data["success"] or len(data["species"]) != 1: + raise + species = next(iter(data["species"].values())) + return species["id"] + except Exception as e: + log.error(f"Couldn't find BirdWeather species ID for {sci_name}_{com_name}: {e}") + raise + + +def query_birdweather_detections( + birdweather_id: str, + species_id: int, + detection_datetime: datetime.datetime, +) -> List[Dict[str, Any]]: + """Query detections from the BirdWeather API for specific station, species and time.""" + detections_url = ( + f"https://app.birdweather.com/api/v1/stations/{birdweather_id}/detections" + ) + try: + resp = requests.get( + url=detections_url, + data={ + "speciesId": species_id, + "from": detection_datetime.isoformat(), + "to": detection_datetime.isoformat(), + }, + timeout=20, + ) + data = resp.json() + if not data["success"]: + raise + return data["detections"] + except Exception as e: + log.error(f"Could not lookup detections from BirdWeather: {e}") + raise + + +def post_soundscape_to_birdweather( + birdweather_id: str, detection_datetime: datetime.datetime, soundscape_file: str +) -> Optional[int]: + """Upload a soundscape file to BirdWeather.""" + soundscape_url = ( + f"https://app.birdweather.com/api/v1/stations/{birdweather_id}/" + f"soundscapes?timestamp={detection_datetime.isoformat()}" + ) + with open(soundscape_file, "rb") as f: + mp3_data = f.read() + gzip_mp3_data = gzip.compress(mp3_data) + try: + resp = requests.post( + url=soundscape_url, + data=gzip_mp3_data, + timeout=20, + headers={ + "Content-Type": "application/octet-stream", + "Content-Encoding": "gzip", + }, + ) + data = resp.json() + if not data.get("success"): + log.error(data.get("message")) + raise + return data["soundscape"]["id"] + except Exception as e: + log.error(f"Cannot POST soundscape: {e}") + return + + +def post_detection_to_birdweather( + detection: Detection, + soundscape_id: str, + soundscape_datetime: datetime.datetime, + birdweather_id: str, + latitude: float, + longitude: float, + model: str +): + """Upload a detection to BirdWeather.""" + + detection_url = f'https://app.birdweather.com/api/v1/stations/{birdweather_id}/detections' + + data = { + 'timestamp': detection.iso8601, + 'lat': latitude, + 'lon': longitude, + 'soundscapeId': soundscape_id, + 'soundscapeStartTime': (detection.start_datetime - soundscape_datetime).seconds, + 'soundscapeEndTime': (detection.stop_datetime - soundscape_datetime).seconds, + 'commonName': detection.common_name, + 'scientificName': detection.scientific_name, + 'algorithm': '2p4' if model == 'BirdNET_GLOBAL_6K_V2.4_Model_FP16' else 'alpha', + 'confidence': detection.confidence + } + + log.debug(data) + try: + response = requests.post(detection_url, json=data, timeout=20) + log.info("Detection POST Response Status - %d", response.status_code) + except Exception as e: + log.error("Cannot POST detection: %s", e) diff --git a/scripts/utils/reporting.py b/scripts/utils/reporting.py index ea8f920c7..9ca458f8e 100644 --- a/scripts/utils/reporting.py +++ b/scripts/utils/reporting.py @@ -1,5 +1,4 @@ import glob -import gzip import json import logging import os @@ -7,10 +6,12 @@ import subprocess from time import sleep +from tzlocal import get_localzone import requests from .helpers import get_settings, ParseFileName, Detection, DB_PATH from .notifications import sendAppriseNotifications +from .birdweather import post_soundscape_to_birdweather, post_detection_to_birdweather log = logging.getLogger(__name__) @@ -160,54 +161,32 @@ def apprise(file: ParseFileName, detections: [Detection]): species_apprised_this_run.append(detection.species) -def bird_weather(file: ParseFileName, detections: [Detection]): +def post_current_detections_to_birdweather(file: ParseFileName, detections: [Detection]): + """Post to BirdWeather detections that were just performed. + + This function relies on the .wav audio file temporarily stored in "StreamData" to post a + soundscape to BirdWeather. + """ conf = get_settings() if conf['BIRDWEATHER_ID'] == "": return if detections: - # POST soundscape to server - soundscape_url = (f'https://app.birdweather.com/api/v1/stations/' - f'{conf["BIRDWEATHER_ID"]}/soundscapes?timestamp={file.iso8601}') - - with open(file.file_name, 'rb') as f: - wav_data = f.read() - gzip_wav_data = gzip.compress(wav_data) - try: - response = requests.post(url=soundscape_url, data=gzip_wav_data, timeout=30, - headers={'Content-Type': 'application/octet-stream', 'Content-Encoding': 'gzip'}) - log.info("Soundscape POST Response Status - %d", response.status_code) - sdata = response.json() - except BaseException as e: - log.error("Cannot POST soundscape: %s", e) - return - if not sdata.get('success'): - log.error(sdata.get('message')) + soundscape_id = post_soundscape_to_birdweather( + conf["BIRDWEATHER_ID"], file.file_date.astimezone(get_localzone()), file.file_name + ) + if soundscape_id is None: return - soundscape_id = sdata['soundscape']['id'] - for detection in detections: - # POST detection to server - detection_url = f'https://app.birdweather.com/api/v1/stations/{conf["BIRDWEATHER_ID"]}/detections' - - data = { - 'timestamp': detection.iso8601, - 'lat': conf['LATITUDE'], - 'lon': conf['LONGITUDE'], - 'soundscapeId': soundscape_id, - 'soundscapeStartTime': (detection.start_datetime - file.file_date).seconds, - 'soundscapeEndTime': (detection.stop_datetime - file.file_date).seconds, - 'commonName': detection.common_name, - 'scientificName': detection.scientific_name, - 'algorithm': '2p4' if conf['MODEL'] == 'BirdNET_GLOBAL_6K_V2.4_Model_FP16' else 'alpha', - 'confidence': detection.confidence, - } - - log.debug(data) - try: - response = requests.post(detection_url, json=data, timeout=20) - log.info("Detection POST Response Status - %d", response.status_code) - except BaseException as e: - log.error("Cannot POST detection: %s", e) + + post_detection_to_birdweather( + detection, + soundscape_id, + file.file_date, + conf["BIRDWEATHER_ID"], + conf['LATITUDE'], + conf['LONGITUDE'], + conf['MODEL'], + ) def heartbeat(): From 6b5182cc7b4524f7925414651b1c6c70110d7121 Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Fri, 6 Dec 2024 10:14:50 +0100 Subject: [PATCH 4/9] Add birdweather_past_publication.py script --- scripts/birdweather_past_publication.py | 155 ++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 scripts/birdweather_past_publication.py diff --git a/scripts/birdweather_past_publication.py b/scripts/birdweather_past_publication.py new file mode 100644 index 000000000..f8188c731 --- /dev/null +++ b/scripts/birdweather_past_publication.py @@ -0,0 +1,155 @@ +"""Publish past detections to BirdWeather.""" + +import datetime +import logging +import os +import sqlite3 +from typing import Optional +import warnings + +import librosa +import pandas as pd +from tzlocal import get_localzone +from utils.helpers import DB_PATH, get_settings, setup_logging, Detection +from utils.birdweather import get_birdweather_species_id, query_birdweather_detections, \ + post_soundscape_to_birdweather, post_detection_to_birdweather + +log = logging.getLogger(os.path.splitext(os.path.basename(os.path.realpath(__file__)))[0]) + + +def get_last_run_time(script_name: str) -> Optional[datetime.datetime]: + """Fetch the last run time for the given script from the database.""" + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + + cursor.execute( + "SELECT last_run FROM scripts_metadata WHERE script_name = ?", (script_name,) + ) + result = cursor.fetchone() + + conn.close() + + if result: + return datetime.datetime.fromisoformat(result[0]) + return None + + +def update_last_run_time(script_name: str): + """Update the last run time for the given script to the current time in the database.""" + conn = sqlite3.connect(DB_PATH) + cursor = conn.cursor() + + current_time = datetime.datetime.now().isoformat() + + cursor.execute( + """ + INSERT INTO scripts_metadata (script_name, last_run) VALUES (?, ?) + ON CONFLICT(script_name) DO UPDATE SET last_run = excluded.last_run; + """, + (script_name, current_time), + ) + + conn.commit() + conn.close() + + +def get_detections_since(start_datetime: datetime.datetime) -> pd.DataFrame: + """Get detections from the database that occurred after the specified date.""" + conn = sqlite3.connect(DB_PATH) + df = pd.read_sql_query( + f"SELECT * from detections WHERE Date > DATE('{start_datetime.strftime('%Y-%m-%d')}')", + conn, + ) + conn.close() + return df + + +def main(): + + conf = get_settings() + if conf["BIRDWEATHER_ID"] == "": + return + + # Get detections since last run (defaults to 7 days if last run time is not found) + last_run_time = get_last_run_time(script_name=os.path.basename(os.path.realpath(__file__))) + if last_run_time is None: + last_run_time = datetime.datetime.now() - datetime.timedelta(days=7) + df = get_detections_since(last_run_time) + + # Loop through recent detections + log.info( + f"Checking if recent detections are present in BirdWeather since {last_run_time}" + ) + for detection_entry in df.itertuples(): + + detection_datetime = datetime.datetime.strptime( + f"{detection_entry.Date} {detection_entry.Time}", "%Y-%m-%d %H:%M:%S" + ).astimezone(get_localzone()) + + try: + # Lookup detections present in BirdWeather at the time of this detection + species_id = get_birdweather_species_id( + detection_entry.Sci_Name, detection_entry.Com_Name + ) + birdweather_detections = query_birdweather_detections( + conf["BIRDWEATHER_ID"], + species_id, + detection_datetime, + ) + except Exception as e: + log.error( + f"Script {os.path.basename(os.path.realpath(__file__))} stopped due to error: {e}" + ) + return + + # This detection is not present in BirdWeather + if birdweather_detections == []: + + log.info(f"Detection not in BirdWeather: {detection_entry.File_Name}") + + # Post extracted audio to BirdWeather as soundscape + extracted_audio_file = os.path.join( + conf["EXTRACTED"], + "By_Date", + detection_datetime.strftime("%Y-%m-%d"), + detection_entry.Com_Name.replace(" ", "_").replace("'", ""), + detection_entry.File_Name, + ) + soundscape_id = post_soundscape_to_birdweather( + conf["BIRDWEATHER_ID"], + detection_datetime, + extracted_audio_file, + ) + + # Get length of extracted audio file, will be useful to post detection to BirdWeather + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=FutureWarning) + soundscape_duration = librosa.get_duration(path=extracted_audio_file) + + # Create an instance of Detection and post it to BirdWeather + # This Detection start and end times are equal to soundscape start and end times, + # because we're using an "extracted" audio file as soundscape + detection = Detection( + detection_datetime, + detection_datetime + datetime.timedelta(seconds=soundscape_duration), + f"{detection_entry.Sci_Name}_{detection_entry.Com_Name}", + detection_entry.Confidence, + ) + post_detection_to_birdweather( + detection, + soundscape_id, + detection_datetime, + conf["BIRDWEATHER_ID"], + conf['LATITUDE'], + conf['LONGITUDE'], + conf['MODEL'], + ) + + update_last_run_time(script_name=os.path.basename(os.path.realpath(__file__))) + + +if __name__ == "__main__": + + setup_logging() + + main() From 03748503670991b3c54f93adaa21339f0fab4e7f Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Fri, 6 Dec 2024 10:15:50 +0100 Subject: [PATCH 5/9] Add scripts_metadata table to SQL DB --- scripts/createdb.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/createdb.sh b/scripts/createdb.sh index 4c66ff383..5e715ef3a 100755 --- a/scripts/createdb.sh +++ b/scripts/createdb.sh @@ -17,6 +17,10 @@ CREATE TABLE IF NOT EXISTS detections ( File_Name VARCHAR(100) NOT NULL); CREATE INDEX "detections_Com_Name" ON "detections" ("Com_Name"); CREATE INDEX "detections_Date_Time" ON "detections" ("Date" DESC, "Time" DESC); +DROP TABLE IF EXISTS scripts_metadata; +CREATE TABLE IF NOT EXISTS scripts_metadata ( + script_name TEXT PRIMARY KEY, + last_run DATETIME); EOF chown $USER:$USER $HOME/BirdNET-Pi/scripts/birds.db chmod g+w $HOME/BirdNET-Pi/scripts/birds.db From 0bdc488a402ebfcd6fc949931ee69eacffa8ae8b Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Sat, 7 Dec 2024 18:03:50 +0100 Subject: [PATCH 6/9] Add past_birdweather_publication in install scripts --- scripts/install_helpers.sh | 30 ++++++++++++++++++++++++++++++ scripts/install_services.sh | 10 +++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/scripts/install_helpers.sh b/scripts/install_helpers.sh index 1a885b3c6..192b4dc18 100644 --- a/scripts/install_helpers.sh +++ b/scripts/install_helpers.sh @@ -70,3 +70,33 @@ install_tmp_mount() { echo "tmp.mount is $STATE, skipping" fi } + +install_birdweather_past_publication() { + cat << EOF > $HOME/BirdNET-Pi/templates/birdweather_past_publication@.service +[Unit] +Description=BirdWeather Publication for %i interface +After=network-online.target +Wants=network-online.target +[Service] +Type=oneshot +User=${USER} +ExecStartPre= /bin/sh -c 'n=0; until curl --silent --head --fail https://app.birdweather.com || [ \$n -ge 30 ]; do n=\$((n+1)); sleep 5; done;' +ExecStart=$PYTHON_VIRTUAL_ENV /usr/local/bin/birdweather_past_publication.py +EOF + cat << EOF > $HOME/BirdNET-Pi/templates/50-birdweather-past-publication +#!/bin/bash +UNIT_NAME="birdweather_past_publication@\$IFACE.service" +# Check if the service is active and then start it +if systemctl is-active --quiet "\$UNIT_NAME"; then + echo "\$UNIT_NAME is already running." +else + echo "Starting \$UNIT_NAME..." + systemctl start "\$UNIT_NAME" +fi +EOF + chmod +x $HOME/BirdNET-Pi/templates/50-birdweather-past-publication + chown root:root $HOME/BirdNET-Pi/templates/50-birdweather-past-publication + ln -sf $HOME/BirdNET-Pi/templates/50-birdweather-past-publication /etc/networkd-dispatcher/routable.d + ln -sf $HOME/BirdNET-Pi/templates/birdweather_past_publication@.service /usr/lib/systemd/system + systemctl enable systemd-networkd +} diff --git a/scripts/install_services.sh b/scripts/install_services.sh index c623ce246..ab06e52a2 100755 --- a/scripts/install_services.sh +++ b/scripts/install_services.sh @@ -20,7 +20,7 @@ install_depends() { apt install -qqy caddy sqlite3 php-sqlite3 php-fpm php-curl php-xml php-zip php icecast2 \ pulseaudio avahi-utils sox libsox-fmt-mp3 alsa-utils ffmpeg \ wget curl unzip bc \ - python3-pip python3-venv lsof net-tools inotify-tools + python3-pip python3-venv lsof net-tools inotify-tools networkd-dispatcher } set_hostname() { @@ -387,6 +387,13 @@ install_weekly_cron() { chown_things() { chown -R $USER:$USER $HOME/Bird* + + # Set ownership to root for the birdweather publication networkd-dispatcher script + BIRDWEATHER_PAST_DISPATCHER_SCRIPT="$HOME/BirdNET-Pi/templates/50-birdweather-past-publication" + if [ -f "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" ]; then + sudo chown root:root "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" + sudo chmod 755 "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" + fi } increase_caddy_timeout() { @@ -409,6 +416,7 @@ install_services() { install_Caddyfile install_avahi_aliases install_birdnet_analysis + install_birdweather_past_publication install_birdnet_stats_service install_recording_service install_custom_recording_service # But does not enable From c21c6d5000dacf3c221515855b9947c1ce3221d3 Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Sat, 7 Dec 2024 18:04:12 +0100 Subject: [PATCH 7/9] Add birdweather_past_publication in update scripts --- scripts/update_birdnet_snippets.sh | 26 ++++++++++++++++++ scripts/update_db.sh | 44 ++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) create mode 100755 scripts/update_db.sh diff --git a/scripts/update_birdnet_snippets.sh b/scripts/update_birdnet_snippets.sh index a6fef3bfe..c0e95197d 100755 --- a/scripts/update_birdnet_snippets.sh +++ b/scripts/update_birdnet_snippets.sh @@ -26,6 +26,9 @@ chmod g+r $HOME # remove world-writable perms chmod -R o-w ~/BirdNET-Pi/templates/* +# update database schema +$my_dir/update_db.sh + APT_UPDATED=0 PIP_UPDATED=0 @@ -147,6 +150,29 @@ if grep -q 'birdnet_server.service' "$HOME/BirdNET-Pi/templates/birdnet_analysis systemctl daemon-reload && restart_services.sh fi + +# Ensure networkd-dispatcher is installed +if ! dpkg -s networkd-dispatcher >/dev/null 2>&1; then + echo "networkd-dispatcher is not installed. Installing it now..." + sudo apt update -qq + sudo apt install -qqy networkd-dispatcher +fi + +# Add BirdWeather past publication service if not already installed +export PYTHON_VIRTUAL_ENV="$HOME/BirdNET-Pi/birdnet/bin/python3" +BIRDWEATHER_PAST_DISPATCHER_SCRIPT="$HOME/BirdNET-Pi/templates/50-birdweather-past-publication" +BIRDWEATHER_PAST_SERVICE_FILE="/usr/lib/systemd/system/birdweather_past_publication@.service" +if [ ! -f "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" ] || [ ! -f "$BIRDWEATHER_PAST_SERVICE_FILE" ]; then + echo "Installing BirdWeather past publication service..." + install_birdweather_past_publication +fi +# Set ownership to root for the birdweather publication networkd-dispatcher script +if [ -f "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" ]; then + sudo chown root:root "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" + sudo chmod 755 "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" +fi + + TMP_MOUNT=$(systemd-escape -p --suffix=mount "$RECS_DIR/StreamData") if ! [ -f "$HOME/BirdNET-Pi/templates/$TMP_MOUNT" ]; then install_birdnet_mount diff --git a/scripts/update_db.sh b/scripts/update_db.sh new file mode 100755 index 000000000..2dd07f40d --- /dev/null +++ b/scripts/update_db.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +DB_PATH="$HOME/BirdNET-Pi/scripts/birds.db" + +echo "Checking database schema for updates" + +# Check if the tables exist +DETECTIONS_TABLE_EXISTS=$(sqlite3 "$DB_PATH" "SELECT name FROM sqlite_master WHERE type='table' AND name='detections';") +SCRIPTS_MTD_TABLE_EXISTS=$(sqlite3 "$DB_PATH" "SELECT name FROM sqlite_master WHERE type='table' AND name='scripts_metadata';") + +if [ -z "$DETECTIONS_TABLE_EXISTS" ]; then + echo "Table 'detections' does not exist. Creating table..." + sqlite3 "$DB_PATH" << EOF + CREATE TABLE IF NOT EXISTS detections ( + Date DATE, + Time TIME, + Sci_Name VARCHAR(100) NOT NULL, + Com_Name VARCHAR(100) NOT NULL, + Confidence FLOAT, + Lat FLOAT, + Lon FLOAT, + Cutoff FLOAT, + Week INT, + Sens FLOAT, + Overlap FLOAT, + File_Name VARCHAR(100) NOT NULL); + CREATE INDEX "detections_Com_Name" ON "detections" ("Com_Name"); + CREATE INDEX "detections_Date_Time" ON "detections" ("Date" DESC, "Time" DESC); +EOF + echo "Table 'detections' created successfully." +elif [ -z "$SCRIPTS_MTD_TABLE_EXISTS" ]; then + echo "Table 'scripts_metadata' does not exist. Creating table..." + sqlite3 "$DB_PATH" << EOF + CREATE TABLE IF NOT EXISTS scripts_metadata ( + script_name TEXT PRIMARY KEY, + last_run DATETIME + ); +EOF + echo "Table 'scripts_metadata' created successfully." +else + echo "Tables 'detections' and 'scripts_metadata' already exist. No changes made." +fi + +echo "Database schema update complete." From 5af42b6f47faeb758f9215163729b0e5390e4f5c Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Sun, 15 Dec 2024 17:03:30 +0100 Subject: [PATCH 8/9] Reduce verbosity for birdweather past publication This suppresses the output of the curl command, used to check if BirdWeather is accessible, by redirecting it to /dev/null. --- scripts/install_helpers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/install_helpers.sh b/scripts/install_helpers.sh index 192b4dc18..f5eaabebd 100644 --- a/scripts/install_helpers.sh +++ b/scripts/install_helpers.sh @@ -80,7 +80,7 @@ Wants=network-online.target [Service] Type=oneshot User=${USER} -ExecStartPre= /bin/sh -c 'n=0; until curl --silent --head --fail https://app.birdweather.com || [ \$n -ge 30 ]; do n=\$((n+1)); sleep 5; done;' +ExecStartPre= /bin/sh -c 'n=0; until curl --silent --head --fail https://app.birdweather.com >/dev/null || [ \$n -ge 30 ]; do n=\$((n+1)); sleep 5; done;' ExecStart=$PYTHON_VIRTUAL_ENV /usr/local/bin/birdweather_past_publication.py EOF cat << EOF > $HOME/BirdNET-Pi/templates/50-birdweather-past-publication From 1cf10763f361e9f4ec9024453406843de8d6d9e1 Mon Sep 17 00:00:00 2001 From: Thibaut Voirand Date: Sun, 15 Dec 2024 17:06:29 +0100 Subject: [PATCH 9/9] Improve reading recent detections in birdweather past publication Take date *and time* into account (and not just the date) when fetching recent detections from the database. --- scripts/birdweather_past_publication.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/birdweather_past_publication.py b/scripts/birdweather_past_publication.py index f8188c731..a2a283686 100644 --- a/scripts/birdweather_past_publication.py +++ b/scripts/birdweather_past_publication.py @@ -56,10 +56,12 @@ def update_last_run_time(script_name: str): def get_detections_since(start_datetime: datetime.datetime) -> pd.DataFrame: """Get detections from the database that occurred after the specified date.""" conn = sqlite3.connect(DB_PATH) - df = pd.read_sql_query( - f"SELECT * from detections WHERE Date > DATE('{start_datetime.strftime('%Y-%m-%d')}')", - conn, + query = ( + "SELECT * FROM detections " + "WHERE DATETIME(Date || ' ' || Time) > " + f"DATETIME('{start_datetime.strftime('%Y-%m-%d %H:%M:%S')}')" ) + df = pd.read_sql_query(query, conn) conn.close() return df