From 2e15099a80b84eb73489712479276b3fbf87b6a8 Mon Sep 17 00:00:00 2001 From: hujun5 Date: Wed, 3 Jul 2024 15:10:49 +0800 Subject: [PATCH] spin_lock: inline spin_lock test: We can use qemu for testing. compiling make distclean -j20; ./tools/configure.sh -l qemu-armv8a:nsh_smp ;make -j20 running qemu-system-aarch64 -cpu cortex-a53 -smp 4 -nographic -machine virt,virtualization=on,gic-version=3 -net none -chardev stdio,id=con,mux=on -serial chardev:con -mon chardev=con,mode=readline -kernel ./nuttx --- arch/arm/src/lpc54xx/lpc54_lowputc.c | 1 + arch/arm/src/lpc54xx/lpc54_serial.c | 1 + arch/arm/src/stm32/stm32_serial.c | 1 + arch/arm/src/stm32f0l0g0/stm32_serial_v1.c | 1 + arch/arm/src/stm32f0l0g0/stm32_serial_v2.c | 1 + arch/arm/src/stm32h7/stm32_serial.c | 1 + arch/arm/src/stm32l4/stm32l4_serial.c | 1 + arch/arm/src/stm32l5/stm32l5_serial.c | 1 + arch/arm/src/stm32u5/stm32_serial.c | 1 + arch/arm/src/stm32wb/stm32wb_serial.c | 1 + arch/arm/src/stm32wl5/stm32wl5_serial.c | 1 + arch/arm/src/tms570/tms570_lowputc.c | 1 + arch/arm/src/xmc4/xmc4_serial.c | 1 + arch/mips/src/pic32mx/pic32mx_serial.c | 1 + arch/mips/src/pic32mz/pic32mz_serial.c | 1 + arch/risc-v/src/bl602/bl602_serial.c | 1 + .../src/esp32c3-legacy/esp32c3_lowputc.c | 1 + arch/sparc/src/bm3803/bm3803-serial.c | 1 + arch/x86_64/src/intel64/intel64_cpustart.c | 1 + drivers/serial/uart_16550.c | 1 + fs/inode/fs_files.c | 19 +- include/nuttx/fs/fs.h | 2 - include/nuttx/spinlock.h | 283 ++++++++- sched/group/group_leave.c | 1 + sched/irq/irq_csection.c | 2 +- sched/semaphore/CMakeLists.txt | 8 - sched/semaphore/Make.defs | 4 - sched/semaphore/spinlock.c | 560 ------------------ sched/signal/sig_findaction.c | 2 + 29 files changed, 305 insertions(+), 596 deletions(-) delete mode 100644 sched/semaphore/spinlock.c diff --git a/arch/arm/src/lpc54xx/lpc54_lowputc.c b/arch/arm/src/lpc54xx/lpc54_lowputc.c index 4360c91b031fb..d9cbd444ff6f3 100644 --- a/arch/arm/src/lpc54xx/lpc54_lowputc.c +++ b/arch/arm/src/lpc54xx/lpc54_lowputc.c @@ -64,6 +64,7 @@ #include #include +#include /**************************************************************************** * Pre-processor Definitions diff --git a/arch/arm/src/lpc54xx/lpc54_serial.c b/arch/arm/src/lpc54xx/lpc54_serial.c index 4d62be6222440..ff6b18b8f555a 100644 --- a/arch/arm/src/lpc54xx/lpc54_serial.c +++ b/arch/arm/src/lpc54xx/lpc54_serial.c @@ -36,6 +36,7 @@ #include #include #include +#include #include diff --git a/arch/arm/src/stm32/stm32_serial.c b/arch/arm/src/stm32/stm32_serial.c index ca5010446cb77..5bcc36099c962 100644 --- a/arch/arm/src/stm32/stm32_serial.c +++ b/arch/arm/src/stm32/stm32_serial.c @@ -38,6 +38,7 @@ #include #include #include +#include #ifdef CONFIG_SERIAL_TERMIOS # include diff --git a/arch/arm/src/stm32f0l0g0/stm32_serial_v1.c b/arch/arm/src/stm32f0l0g0/stm32_serial_v1.c index 770aacb4dc321..9799dc81b85c9 100644 --- a/arch/arm/src/stm32f0l0g0/stm32_serial_v1.c +++ b/arch/arm/src/stm32f0l0g0/stm32_serial_v1.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_TERMIOS diff --git a/arch/arm/src/stm32f0l0g0/stm32_serial_v2.c b/arch/arm/src/stm32f0l0g0/stm32_serial_v2.c index 2c2a701ee8db3..11009babdf1f3 100644 --- a/arch/arm/src/stm32f0l0g0/stm32_serial_v2.c +++ b/arch/arm/src/stm32f0l0g0/stm32_serial_v2.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_TERMIOS diff --git a/arch/arm/src/stm32h7/stm32_serial.c b/arch/arm/src/stm32h7/stm32_serial.c index ee33b6a5bc896..5bf37e7e0d168 100644 --- a/arch/arm/src/stm32h7/stm32_serial.c +++ b/arch/arm/src/stm32h7/stm32_serial.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/src/stm32l4/stm32l4_serial.c b/arch/arm/src/stm32l4/stm32l4_serial.c index 648fc995ba906..c3d65693dbb7e 100644 --- a/arch/arm/src/stm32l4/stm32l4_serial.c +++ b/arch/arm/src/stm32l4/stm32l4_serial.c @@ -37,6 +37,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_TERMIOS diff --git a/arch/arm/src/stm32l5/stm32l5_serial.c b/arch/arm/src/stm32l5/stm32l5_serial.c index b483d1835fb3d..d529ddaf330b6 100644 --- a/arch/arm/src/stm32l5/stm32l5_serial.c +++ b/arch/arm/src/stm32l5/stm32l5_serial.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_TERMIOS diff --git a/arch/arm/src/stm32u5/stm32_serial.c b/arch/arm/src/stm32u5/stm32_serial.c index 97f90300e2668..9394ee9f496d3 100644 --- a/arch/arm/src/stm32u5/stm32_serial.c +++ b/arch/arm/src/stm32u5/stm32_serial.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_TERMIOS diff --git a/arch/arm/src/stm32wb/stm32wb_serial.c b/arch/arm/src/stm32wb/stm32wb_serial.c index b83a23ff693d6..d1d8bb3ced44c 100644 --- a/arch/arm/src/stm32wb/stm32wb_serial.c +++ b/arch/arm/src/stm32wb/stm32wb_serial.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include diff --git a/arch/arm/src/stm32wl5/stm32wl5_serial.c b/arch/arm/src/stm32wl5/stm32wl5_serial.c index 037bbbcfa5bd5..fb34ef7519db8 100644 --- a/arch/arm/src/stm32wl5/stm32wl5_serial.c +++ b/arch/arm/src/stm32wl5/stm32wl5_serial.c @@ -38,6 +38,7 @@ #include #include #include +#include #include #ifdef CONFIG_SERIAL_TERMIOS diff --git a/arch/arm/src/tms570/tms570_lowputc.c b/arch/arm/src/tms570/tms570_lowputc.c index 9f7232460b3d4..87d6dd5709362 100644 --- a/arch/arm/src/tms570/tms570_lowputc.c +++ b/arch/arm/src/tms570/tms570_lowputc.c @@ -52,6 +52,7 @@ #include #include #include +#include #include "arm_internal.h" #include "hardware/tms570_sci.h" diff --git a/arch/arm/src/xmc4/xmc4_serial.c b/arch/arm/src/xmc4/xmc4_serial.c index 6e0644a1a76cb..a2b060d08085b 100644 --- a/arch/arm/src/xmc4/xmc4_serial.c +++ b/arch/arm/src/xmc4/xmc4_serial.c @@ -36,6 +36,7 @@ #include #include #include +#include #include diff --git a/arch/mips/src/pic32mx/pic32mx_serial.c b/arch/mips/src/pic32mx/pic32mx_serial.c index 1726f628d5432..7f35a82500811 100644 --- a/arch/mips/src/pic32mx/pic32mx_serial.c +++ b/arch/mips/src/pic32mx/pic32mx_serial.c @@ -40,6 +40,7 @@ #include #include #include +#include #include diff --git a/arch/mips/src/pic32mz/pic32mz_serial.c b/arch/mips/src/pic32mz/pic32mz_serial.c index 8f5353010e20b..f1c81bceeb86b 100644 --- a/arch/mips/src/pic32mz/pic32mz_serial.c +++ b/arch/mips/src/pic32mz/pic32mz_serial.c @@ -41,6 +41,7 @@ #include #include #include +#include #include diff --git a/arch/risc-v/src/bl602/bl602_serial.c b/arch/risc-v/src/bl602/bl602_serial.c index f9b5d033349e2..f03ab039e0b71 100644 --- a/arch/risc-v/src/bl602/bl602_serial.c +++ b/arch/risc-v/src/bl602/bl602_serial.c @@ -37,6 +37,7 @@ #include #include #include +#include #include "bl602_lowputc.h" #include "bl602_gpio.h" diff --git a/arch/risc-v/src/esp32c3-legacy/esp32c3_lowputc.c b/arch/risc-v/src/esp32c3-legacy/esp32c3_lowputc.c index d5a36999b506d..4620942e475c5 100644 --- a/arch/risc-v/src/esp32c3-legacy/esp32c3_lowputc.c +++ b/arch/risc-v/src/esp32c3-legacy/esp32c3_lowputc.c @@ -25,6 +25,7 @@ #include #include #include +#include #include #include diff --git a/arch/sparc/src/bm3803/bm3803-serial.c b/arch/sparc/src/bm3803/bm3803-serial.c index b6911025f53b2..192065d2cccb8 100644 --- a/arch/sparc/src/bm3803/bm3803-serial.c +++ b/arch/sparc/src/bm3803/bm3803-serial.c @@ -41,6 +41,7 @@ #include #include #include +#include #include diff --git a/arch/x86_64/src/intel64/intel64_cpustart.c b/arch/x86_64/src/intel64/intel64_cpustart.c index f8f2a703b7844..c766ebfe1c000 100644 --- a/arch/x86_64/src/intel64/intel64_cpustart.c +++ b/arch/x86_64/src/intel64/intel64_cpustart.c @@ -30,6 +30,7 @@ #include #include #include +#include #include "init/init.h" diff --git a/drivers/serial/uart_16550.c b/drivers/serial/uart_16550.c index 747383a6f51f0..e03fee887def8 100644 --- a/drivers/serial/uart_16550.c +++ b/drivers/serial/uart_16550.c @@ -42,6 +42,7 @@ #include #include #include +#include #include diff --git a/fs/inode/fs_files.c b/fs/inode/fs_files.c index a95f31847a287..f5177c8246390 100644 --- a/fs/inode/fs_files.c +++ b/fs/inode/fs_files.c @@ -41,6 +41,7 @@ #include #include #include +#include #ifdef CONFIG_FDSAN # include @@ -66,11 +67,11 @@ static FAR struct file *files_fget_by_index(FAR struct filelist *list, FAR struct file *filep; irqstate_t flags; - flags = spin_lock_irqsave(&list->fl_lock); + flags = spin_lock_irqsave(NULL); filep = &list->fl_files[l1][l2]; - spin_unlock_irqrestore(&list->fl_lock, flags); + spin_unlock_irqrestore(NULL, flags); return filep; } @@ -125,7 +126,7 @@ static int files_extend(FAR struct filelist *list, size_t row) } while (++i < row); - flags = spin_lock_irqsave(&list->fl_lock); + flags = spin_lock_irqsave(NULL); /* To avoid race condition, if the file list is updated by other threads * and list rows is greater or equal than temp list, @@ -134,7 +135,7 @@ static int files_extend(FAR struct filelist *list, size_t row) if (orig_rows != list->fl_rows && list->fl_rows >= row) { - spin_unlock_irqrestore(&list->fl_lock, flags); + spin_unlock_irqrestore(NULL, flags); for (j = orig_rows; j < i; j++) { @@ -156,7 +157,7 @@ static int files_extend(FAR struct filelist *list, size_t row) list->fl_files = files; list->fl_rows = row; - spin_unlock_irqrestore(&list->fl_lock, flags); + spin_unlock_irqrestore(NULL, flags); if (tmp != NULL && tmp != &list->fl_prefile) { @@ -484,13 +485,13 @@ int file_allocate_from_tcb(FAR struct tcb_s *tcb, FAR struct inode *inode, /* Find free file */ - flags = spin_lock_irqsave(&list->fl_lock); + flags = spin_lock_irqsave(NULL); for (; ; i++, j = 0) { if (i >= list->fl_rows) { - spin_unlock_irqrestore(&list->fl_lock, flags); + spin_unlock_irqrestore(NULL, flags); ret = files_extend(list, i + 1); if (ret < 0) @@ -498,7 +499,7 @@ int file_allocate_from_tcb(FAR struct tcb_s *tcb, FAR struct inode *inode, return ret; } - flags = spin_lock_irqsave(&list->fl_lock); + flags = spin_lock_irqsave(NULL); } do @@ -518,7 +519,7 @@ int file_allocate_from_tcb(FAR struct tcb_s *tcb, FAR struct inode *inode, } found: - spin_unlock_irqrestore(&list->fl_lock, flags); + spin_unlock_irqrestore(NULL, flags); if (addref) { diff --git a/include/nuttx/fs/fs.h b/include/nuttx/fs/fs.h index bca2f8dd190e3..cd9beebd7d2df 100644 --- a/include/nuttx/fs/fs.h +++ b/include/nuttx/fs/fs.h @@ -37,7 +37,6 @@ #include #include -#include #include #include #include @@ -490,7 +489,6 @@ struct file struct filelist { - spinlock_t fl_lock; /* Manage access to the file list */ uint8_t fl_rows; /* The number of rows of fl_files array */ FAR struct file **fl_files; /* The pointer of two layer file descriptors array */ diff --git a/include/nuttx/spinlock.h b/include/nuttx/spinlock.h index 4bb2203fcfe64..d3bce433a2369 100644 --- a/include/nuttx/spinlock.h +++ b/include/nuttx/spinlock.h @@ -28,8 +28,30 @@ #include #include +#include #include +#if defined(CONFIG_TICKET_SPINLOCK) || defined(CONFIG_RW_SPINLOCK) +# if !defined(__cplusplus) +# include +# define CONFIG_HAVE_INLINE_SPINLOCK +# elif defined(__has_include) && __has_include() +extern "C++" +{ +# include +# define CONFIG_HAVE_INLINE_SPINLOCK + using std::atomic_int; + using std::atomic_load; + using std::atomic_fetch_add; + using std::atomic_fetch_sub; + using std::atomic_compare_exchange_strong; +} +# endif +#else +# define CONFIG_HAVE_INLINE_SPINLOCK +#endif + +#include #include #undef EXTERN @@ -184,6 +206,8 @@ static inline spinlock_t up_testset(FAR volatile spinlock_t *lock) /* void spin_lock_init(FAR spinlock_t *lock); */ #define spin_lock_init(l) do { *(l) = SP_UNLOCKED; } while (0) +#ifdef CONFIG_HAVE_INLINE_SPINLOCK + /**************************************************************************** * Name: spin_lock * @@ -207,7 +231,35 @@ static inline spinlock_t up_testset(FAR volatile spinlock_t *lock) * ****************************************************************************/ -void spin_lock(FAR volatile spinlock_t *lock); +static inline_function void spin_lock(FAR volatile spinlock_t *lock) +{ +#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS + struct tcb_s *tcb = this_task(); + + /* Notify that we are waiting for a spinlock */ + + sched_note_spinlock(tcb, lock, NOTE_SPINLOCK_LOCK); +#endif + +#ifdef CONFIG_TICKET_SPINLOCK + unsigned short ticket = + atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.next, 1); + while (atomic_load((FAR atomic_ushort *)&lock->tickets.owner) != ticket) +#else /* CONFIG_SPINLOCK */ + while (up_testset(lock) == SP_LOCKED) +#endif + { + SP_DSB(); + SP_WFE(); + } + +#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS + /* Notify that we have the spinlock */ + + sched_note_spinlock(tcb, lock, NOTE_SPINLOCK_LOCKED); +#endif + SP_DMB(); +} /**************************************************************************** * Name: spin_lock_wo_note @@ -231,7 +283,22 @@ void spin_lock(FAR volatile spinlock_t *lock); * ****************************************************************************/ -void spin_lock_wo_note(FAR volatile spinlock_t *lock); +static inline_function void spin_lock_wo_note(FAR volatile spinlock_t *lock) +{ +#ifdef CONFIG_TICKET_SPINLOCK + unsigned short ticket = + atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.next, 1); + while (atomic_load((FAR atomic_ushort *)&lock->tickets.owner) != ticket) +#else /* CONFIG_TICKET_SPINLOCK */ + while (up_testset(lock) == SP_LOCKED) +#endif + { + SP_DSB(); + SP_WFE(); + } + + SP_DMB(); +} /**************************************************************************** * Name: spin_trylock @@ -252,7 +319,57 @@ void spin_lock_wo_note(FAR volatile spinlock_t *lock); * ****************************************************************************/ -bool spin_trylock(FAR volatile spinlock_t *lock); +static inline_function bool spin_trylock(FAR volatile spinlock_t *lock) +{ +#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS + struct tcb_s *tcb = this_task(); + + /* Notify that we are waiting for a spinlock */ + + sched_note_spinlock(tcb, lock, NOTE_SPINLOCK_LOCK); +#endif + +#ifdef CONFIG_TICKET_SPINLOCK + unsigned short ticket = + atomic_load((FAR atomic_ushort *)&lock->tickets.next); + + spinlock_t old = + { + { + ticket, ticket + } + }; + + spinlock_t new = + { + { + ticket, ticket + 1 + } + }; + + if (!atomic_compare_exchange_strong((FAR atomic_uint *)&lock->value, + &old.value, new.value)) +#else /* CONFIG_TICKET_SPINLOCK */ + if (up_testset(lock) == SP_LOCKED) +#endif /* CONFIG_TICKET_SPINLOCK */ + { +#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS + /* Notify that we abort for a spinlock */ + + sched_note_spinlock(tcb, lock, NOTE_SPINLOCK_ABORT); +#endif + SP_DSB(); + return false; + } + +#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS + /* Notify that we have the spinlock */ + + sched_note_spinlock(tcb, lock, NOTE_SPINLOCK_LOCKED); +#endif + SP_DMB(); + return true; +} /**************************************************************************** * Name: spin_trylock_wo_note @@ -276,7 +393,40 @@ bool spin_trylock(FAR volatile spinlock_t *lock); * ****************************************************************************/ -bool spin_trylock_wo_note(FAR volatile spinlock_t *lock); +static inline_function bool +spin_trylock_wo_note(FAR volatile spinlock_t *lock) +{ +#ifdef CONFIG_TICKET_SPINLOCK + unsigned short ticket = + atomic_load((FAR atomic_ushort *)&lock->tickets.next); + + spinlock_t old = + { + { + ticket, ticket + } + }; + + spinlock_t new = + { + { + ticket, ticket + 1 + } + }; + + if (!atomic_compare_exchange_strong((FAR atomic_uint *)&lock->value, + &old.value, new.value)) +#else /* CONFIG_TICKET_SPINLOCK */ + if (up_testset(lock) == SP_LOCKED) +#endif /* CONFIG_TICKET_SPINLOCK */ + { + SP_DSB(); + return false; + } + + SP_DMB(); + return true; +} /**************************************************************************** * Name: spin_unlock @@ -296,7 +446,23 @@ bool spin_trylock_wo_note(FAR volatile spinlock_t *lock); ****************************************************************************/ #ifdef __SP_UNLOCK_FUNCTION -void spin_unlock(FAR volatile spinlock_t *lock); +static inline_function void spin_unlock(FAR volatile spinlock_t *lock) +{ +# ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS + /* Notify that we are unlocking the spinlock */ + + sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_UNLOCK); +# endif + + SP_DMB(); +# ifdef CONFIG_TICKET_SPINLOCK + atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.owner, 1); +# else + *lock = SP_UNLOCKED; +# endif + SP_DSB(); + SP_SEV(); +} #else # define spin_unlock(l) do { *(l) = SP_UNLOCKED; } while (0) #endif @@ -321,7 +487,18 @@ void spin_unlock(FAR volatile spinlock_t *lock); * ****************************************************************************/ -void spin_unlock_wo_note(FAR volatile spinlock_t *lock); +static inline_function void +spin_unlock_wo_note(FAR volatile spinlock_t *lock) +{ + SP_DMB(); +#ifdef CONFIG_TICKET_SPINLOCK + atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.owner, 1); +#else + *lock = SP_UNLOCKED; +#endif + SP_DSB(); + SP_SEV(); +} /**************************************************************************** * Name: spin_is_locked @@ -502,7 +679,26 @@ void spin_unlock_irqrestore_wo_note(FAR spinlock_t *lock, irqstate_t flags); * ****************************************************************************/ -void read_lock(FAR volatile rwlock_t *lock); +static inline_function void read_lock(FAR volatile rwlock_t *lock) +{ + while (true) + { + int old = atomic_load((FAR atomic_int *)lock); + if (old <= RW_SP_WRITE_LOCKED) + { + DEBUGASSERT(old == RW_SP_WRITE_LOCKED); + SP_DSB(); + SP_WFE(); + } + else if(atomic_compare_exchange_strong((FAR atomic_int *)lock, + &old, old + 1)) + { + break; + } + } + + SP_DMB(); +} /**************************************************************************** * Name: read_trylock @@ -528,7 +724,26 @@ void read_lock(FAR volatile rwlock_t *lock); * ****************************************************************************/ -bool read_trylock(FAR volatile rwlock_t *lock); +static inline_function bool read_trylock(FAR volatile rwlock_t *lock) +{ + while (true) + { + int old = atomic_load((FAR atomic_int *)lock); + if (old <= RW_SP_WRITE_LOCKED) + { + DEBUGASSERT(old == RW_SP_WRITE_LOCKED); + return false; + } + else if (atomic_compare_exchange_strong((FAR atomic_int *)lock, + &old, old + 1)) + { + break; + } + } + + SP_DMB(); + return true; +} /**************************************************************************** * Name: read_unlock @@ -547,7 +762,15 @@ bool read_trylock(FAR volatile rwlock_t *lock); * ****************************************************************************/ -void read_unlock(FAR volatile rwlock_t *lock); +static inline_function void read_unlock(FAR volatile rwlock_t *lock) +{ + DEBUGASSERT(atomic_load((FAR atomic_int *)lock) >= RW_SP_READ_LOCKED); + + SP_DMB(); + atomic_fetch_sub((FAR atomic_int *)lock, 1); + SP_DSB(); + SP_SEV(); +} /**************************************************************************** * Name: write_lock @@ -574,7 +797,19 @@ void read_unlock(FAR volatile rwlock_t *lock); * ****************************************************************************/ -void write_lock(FAR volatile rwlock_t *lock); +static inline_function void write_lock(FAR volatile rwlock_t *lock) +{ + int zero = RW_SP_UNLOCKED; + + while (!atomic_compare_exchange_strong((FAR atomic_int *)lock, + &zero, RW_SP_WRITE_LOCKED)) + { + SP_DSB(); + SP_WFE(); + } + + SP_DMB(); +} /**************************************************************************** * Name: write_trylock @@ -601,7 +836,20 @@ void write_lock(FAR volatile rwlock_t *lock); * ****************************************************************************/ -bool write_trylock(FAR volatile rwlock_t *lock); +static inline_function bool write_trylock(FAR volatile rwlock_t *lock) +{ + int zero = RW_SP_UNLOCKED; + + if (atomic_compare_exchange_strong((FAR atomic_int *)lock, + &zero, RW_SP_WRITE_LOCKED)) + { + SP_DMB(); + return true; + } + + SP_DSB(); + return false; +} /**************************************************************************** * Name: write_unlock @@ -620,7 +868,17 @@ bool write_trylock(FAR volatile rwlock_t *lock); * ****************************************************************************/ -void write_unlock(FAR volatile rwlock_t *lock); +static inline_function void write_unlock(FAR volatile rwlock_t *lock) +{ + /* Ensure this cpu already get write lock */ + + DEBUGASSERT(atomic_load((FAR atomic_int *)lock) == RW_SP_WRITE_LOCKED); + + SP_DMB(); + atomic_store((FAR atomic_int *)lock, RW_SP_UNLOCKED); + SP_DSB(); + SP_SEV(); +} /**************************************************************************** * Name: read_lock_irqsave @@ -767,6 +1025,7 @@ void write_unlock_irqrestore(FAR rwlock_t *lock, irqstate_t flags); #endif #endif /* CONFIG_RW_SPINLOCK */ +#endif /* CONFIG_HAVE_INLINE_SPINLOCK */ #undef EXTERN #if defined(__cplusplus) diff --git a/sched/group/group_leave.c b/sched/group/group_leave.c index fe452f27c4c68..4cab22c46ad62 100644 --- a/sched/group/group_leave.c +++ b/sched/group/group_leave.c @@ -34,6 +34,7 @@ #include #include #include +#include #ifdef CONFIG_BINFMT_LOADABLE # include diff --git a/sched/irq/irq_csection.c b/sched/irq/irq_csection.c index 224358642dc79..4c20386094154 100644 --- a/sched/irq/irq_csection.c +++ b/sched/irq/irq_csection.c @@ -117,7 +117,7 @@ volatile uint8_t g_cpu_nestcount[CONFIG_SMP_NCPUS]; ****************************************************************************/ #ifdef CONFIG_SMP -static bool irq_waitlock(int cpu) +static inline_function bool irq_waitlock(int cpu) { #ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS FAR struct tcb_s *tcb = current_task(cpu); diff --git a/sched/semaphore/CMakeLists.txt b/sched/semaphore/CMakeLists.txt index 9ed4e0bdfab20..8d378e04b0286 100644 --- a/sched/semaphore/CMakeLists.txt +++ b/sched/semaphore/CMakeLists.txt @@ -36,12 +36,4 @@ if(CONFIG_PRIORITY_INHERITANCE) list(APPEND CSRCS sem_initialize.c sem_holder.c sem_setprotocol.c) endif() -if(CONFIG_SPINLOCK) - list(APPEND CSRCS spinlock.c) -endif() - -if(CONFIG_TICKET_SPINLOCK) - list(APPEND CSRCS spinlock.c) -endif() - target_sources(sched PRIVATE ${CSRCS}) diff --git a/sched/semaphore/Make.defs b/sched/semaphore/Make.defs index 71dcc26d1247a..f2a9e7ffb4f06 100644 --- a/sched/semaphore/Make.defs +++ b/sched/semaphore/Make.defs @@ -28,10 +28,6 @@ ifeq ($(CONFIG_PRIORITY_INHERITANCE),y) CSRCS += sem_initialize.c sem_holder.c sem_setprotocol.c endif -ifeq ($(CONFIG_SPINLOCK),y) -CSRCS += spinlock.c -endif - # Include semaphore build support DEPPATH += --dep-path semaphore diff --git a/sched/semaphore/spinlock.c b/sched/semaphore/spinlock.c deleted file mode 100644 index 49eeb752dc763..0000000000000 --- a/sched/semaphore/spinlock.c +++ /dev/null @@ -1,560 +0,0 @@ -/**************************************************************************** - * sched/semaphore/spinlock.c - * - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. The - * ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the - * License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - * - ****************************************************************************/ - -/**************************************************************************** - * Included Files - ****************************************************************************/ - -#include - -#include -#include -#include -#include - -#include -#include -#include - -#if defined(CONFIG_TICKET_SPINLOCK) || defined(CONFIG_RW_SPINLOCK) -# include -#endif - -#include "sched/sched.h" - -#if defined(CONFIG_SPINLOCK) || defined(CONFIG_TICKET_SPINLOCK) - -/**************************************************************************** - * Public Functions - ****************************************************************************/ - -/**************************************************************************** - * Name: spin_lock - * - * Description: - * If this CPU does not already hold the spinlock, then loop until the - * spinlock is successfully locked. - * - * This implementation is non-reentrant and is prone to deadlocks in - * the case that any logic on the same CPU attempts to take the lock - * more than once. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * None. When the function returns, the spinlock was successfully locked - * by this CPU. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void spin_lock(FAR volatile spinlock_t *lock) -{ -#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS - /* Notify that we are waiting for a spinlock */ - - sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_LOCK); -#endif - -#ifdef CONFIG_TICKET_SPINLOCK - unsigned short ticket = - atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.next, 1); - while (atomic_load((FAR atomic_ushort *)&lock->tickets.owner) != ticket) -#else /* CONFIG_SPINLOCK */ - while (up_testset(lock) == SP_LOCKED) -#endif - { - SP_DSB(); - SP_WFE(); - } - -#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS - /* Notify that we have the spinlock */ - - sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_LOCKED); -#endif - SP_DMB(); -} - -/**************************************************************************** - * Name: spin_lock_wo_note - * - * Description: - * If this CPU does not already hold the spinlock, then loop until the - * spinlock is successfully locked. - * - * This implementation is the same as the above spin_lock() except that - * it does not perform instrumentation logic. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * None. When the function returns, the spinlock was successfully locked - * by this CPU. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void spin_lock_wo_note(FAR volatile spinlock_t *lock) -{ -#ifdef CONFIG_TICKET_SPINLOCK - unsigned short ticket = - atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.next, 1); - while (atomic_load((FAR atomic_ushort *)&lock->tickets.owner) != ticket) -#else /* CONFIG_TICKET_SPINLOCK */ - while (up_testset(lock) == SP_LOCKED) -#endif - { - SP_DSB(); - SP_WFE(); - } - - SP_DMB(); -} - -/**************************************************************************** - * Name: spin_trylock - * - * Description: - * Try once to lock the spinlock. Do not wait if the spinlock is already - * locked. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * false - Failure, the spinlock was already locked - * true - Success, the spinlock was successfully locked - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -bool spin_trylock(FAR volatile spinlock_t *lock) -{ -#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS - /* Notify that we are waiting for a spinlock */ - - sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_LOCK); -#endif - -#ifdef CONFIG_TICKET_SPINLOCK - unsigned short ticket = - atomic_load((FAR atomic_ushort *)&lock->tickets.next); - - spinlock_t old = - { - { - ticket, ticket - } - }; - - spinlock_t new = - { - { - ticket, ticket + 1 - } - }; - - if (!atomic_compare_exchange_strong((FAR atomic_uint *)&lock->value, - &old.value, new.value)) -#else /* CONFIG_TICKET_SPINLOCK */ - if (up_testset(lock) == SP_LOCKED) -#endif /* CONFIG_TICKET_SPINLOCK */ - { -#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS - /* Notify that we abort for a spinlock */ - - sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_ABORT); -#endif - SP_DSB(); - return false; - } - -#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS - /* Notify that we have the spinlock */ - - sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_LOCKED); -#endif - SP_DMB(); - return true; -} - -/**************************************************************************** - * Name: spin_trylock_wo_note - * - * Description: - * Try once to lock the spinlock. Do not wait if the spinlock is already - * locked. - * - * This implementation is the same as the above spin_trylock() except that - * it does not perform instrumentation logic. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * false - Failure, the spinlock was already locked - * true - Success, the spinlock was successfully locked - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -bool spin_trylock_wo_note(FAR volatile spinlock_t *lock) -{ -#ifdef CONFIG_TICKET_SPINLOCK - unsigned short ticket = - atomic_load((FAR atomic_ushort *)&lock->tickets.next); - - spinlock_t old = - { - { - ticket, ticket - } - }; - - spinlock_t new = - { - { - ticket, ticket + 1 - } - }; - - if (!atomic_compare_exchange_strong((FAR atomic_uint *)&lock->value, - &old.value, new.value)) -#else /* CONFIG_TICKET_SPINLOCK */ - if (up_testset(lock) == SP_LOCKED) -#endif /* CONFIG_TICKET_SPINLOCK */ - { - SP_DSB(); - return false; - } - - SP_DMB(); - return true; -} - -/**************************************************************************** - * Name: spin_unlock - * - * Description: - * Release one count on a non-reentrant spinlock. - * - * Input Parameters: - * lock - A reference to the spinlock object to unlock. - * - * Returned Value: - * None. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -#ifdef __SP_UNLOCK_FUNCTION -void spin_unlock(FAR volatile spinlock_t *lock) -{ -#ifdef CONFIG_SCHED_INSTRUMENTATION_SPINLOCKS - /* Notify that we are unlocking the spinlock */ - - sched_note_spinlock(this_task(), lock, NOTE_SPINLOCK_UNLOCK); -#endif - - SP_DMB(); -#ifdef CONFIG_TICKET_SPINLOCK - atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.owner, 1); -#else - *lock = SP_UNLOCKED; -#endif - SP_DSB(); - SP_SEV(); -} -#endif - -/**************************************************************************** - * Name: spin_unlock_wo_note - * - * Description: - * Release one count on a non-reentrant spinlock. - * - * This implementation is the same as the above spin_unlock() except that - * it does not perform instrumentation logic. - * - * Input Parameters: - * lock - A reference to the spinlock object to unlock. - * - * Returned Value: - * None. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void spin_unlock_wo_note(FAR volatile spinlock_t *lock) -{ - SP_DMB(); -#ifdef CONFIG_TICKET_SPINLOCK - atomic_fetch_add((FAR atomic_ushort *)&lock->tickets.owner, 1); -#else - *lock = SP_UNLOCKED; -#endif - SP_DSB(); - SP_SEV(); -} - -#ifdef CONFIG_RW_SPINLOCK - -/**************************************************************************** - * Name: read_lock - * - * Description: - * If this task does not already hold the spinlock, then loop until the - * spinlock is successfully locked. - * - * This implementation is non-reentrant and set a bit of lock. - * - * The priority of reader is higher than writter if a reader hold the - * lock, a new reader can get its lock but writer can't get this lock. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * None. When the function returns, the spinlock was successfully locked - * by this CPU. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void read_lock(FAR volatile rwlock_t *lock) -{ - while (true) - { - int old = atomic_load((FAR atomic_int *)lock); - if (old <= RW_SP_WRITE_LOCKED) - { - DEBUGASSERT(old == RW_SP_WRITE_LOCKED); - SP_DSB(); - SP_WFE(); - } - else if(atomic_compare_exchange_strong((FAR atomic_int *)lock, - &old, old + 1)) - { - break; - } - } - - SP_DMB(); -} - -/**************************************************************************** - * Name: read_trylock - * - * Description: - * If this task does not already hold the spinlock, then try to get the - * lock. - * - * This implementation is non-reentrant and set a bit of lock. - * - * The priority of reader is higher than writter if a reader hold the - * lock, a new reader can get its lock but writer can't get this lock. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * false - Failure, the spinlock was already locked - * true - Success, the spinlock was successfully locked - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -bool read_trylock(FAR volatile rwlock_t *lock) -{ - while (true) - { - int old = atomic_load((FAR atomic_int *)lock); - if (old <= RW_SP_WRITE_LOCKED) - { - DEBUGASSERT(old == RW_SP_WRITE_LOCKED); - return false; - } - else if (atomic_compare_exchange_strong((FAR atomic_int *)lock, - &old, old + 1)) - { - break; - } - } - - SP_DMB(); - return true; -} - -/**************************************************************************** - * Name: read_unlock - * - * Description: - * Release a bit on a non-reentrant spinlock. - * - * Input Parameters: - * lock - A reference to the spinlock object to unlock. - * - * Returned Value: - * None. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void read_unlock(FAR volatile rwlock_t *lock) -{ - DEBUGASSERT(atomic_load((FAR atomic_int *)lock) >= RW_SP_READ_LOCKED); - - SP_DMB(); - atomic_fetch_sub((FAR atomic_int *)lock, 1); - SP_DSB(); - SP_SEV(); -} - -/**************************************************************************** - * Name: write_lock - * - * Description: - * If this task does not already hold the spinlock, then loop until the - * spinlock is successfully locked. - * - * This implementation is non-reentrant and set all bit on lock to avoid - * readers and writers. - * - * The priority of reader is higher than writter if a reader hold the - * lock, a new reader can get its lock but writer can't get this lock. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * None. When the function returns, the spinlock was successfully locked - * by this CPU. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void write_lock(FAR volatile rwlock_t *lock) -{ - int zero = RW_SP_UNLOCKED; - - while (!atomic_compare_exchange_strong((FAR atomic_int *)lock, - &zero, RW_SP_WRITE_LOCKED)) - { - SP_DSB(); - SP_WFE(); - } - - SP_DMB(); -} - -/**************************************************************************** - * Name: write_trylock - * - * Description: - * If this task does not already hold the spinlock, then loop until the - * spinlock is successfully locked. - * - * This implementation is non-reentrant and set all bit on lock to avoid - * readers and writers. - * - * The priority of reader is higher than writter if a reader hold the - * lock, a new reader can get its lock but writer can't get this lock. - * - * Input Parameters: - * lock - A reference to the spinlock object to lock. - * - * Returned Value: - * false - Failure, the spinlock was already locked - * true - Success, the spinlock was successfully locked - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -bool write_trylock(FAR volatile rwlock_t *lock) -{ - int zero = RW_SP_UNLOCKED; - - if (atomic_compare_exchange_strong((FAR atomic_int *)lock, - &zero, RW_SP_WRITE_LOCKED)) - { - SP_DMB(); - return true; - } - - SP_DSB(); - return false; -} - -/**************************************************************************** - * Name: write_unlock - * - * Description: - * Release write lock on a non-reentrant spinlock. - * - * Input Parameters: - * lock - A reference to the spinlock object to unlock. - * - * Returned Value: - * None. - * - * Assumptions: - * Not running at the interrupt level. - * - ****************************************************************************/ - -void write_unlock(FAR volatile rwlock_t *lock) -{ - /* Ensure this cpu already get write lock */ - - DEBUGASSERT(atomic_load((FAR atomic_int *)lock) == RW_SP_WRITE_LOCKED); - - SP_DMB(); - atomic_store((FAR atomic_int *)lock, RW_SP_UNLOCKED); - SP_DSB(); - SP_SEV(); -} - -#endif /* CONFIG_RW_SPINLOCK */ -#endif /* CONFIG_SPINLOCK */ diff --git a/sched/signal/sig_findaction.c b/sched/signal/sig_findaction.c index a0ef73171ef81..0c24bd761e240 100644 --- a/sched/signal/sig_findaction.c +++ b/sched/signal/sig_findaction.c @@ -23,6 +23,8 @@ ****************************************************************************/ #include + +#include #include "signal/signal.h" /****************************************************************************