diff --git a/Makefile.am b/Makefile.am
index c08b3529..705d469f 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -33,7 +33,7 @@ check_PROGRAMS =
 AM_CPPFLAGS = -DPKGLIBDIR="\"$(pkglibdir)\""
 # That can be trimmed down later when adapting the sources to
 # use relative paths for includes.
-# Watch out for generated headers (that's why top_builddir is also present).
+# TODO: Actually do more relative includes.
 AM_CPPFLAGS += \
   $(LTDLINCL) \
   -I$(top_srcdir)/src \
@@ -64,6 +64,9 @@ endif
 include src/Makemodule.am
 include doc/Makemodule.am
 
+# Always use our set of API headers.
+AM_CPPFLAGS+= -I$(top_srcdir)/src/include
+
 if NEED_MAINLIB
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = 
@@ -71,15 +74,12 @@ pkgconfig_DATA =
 # Pkg-config files and library includes.
 if BUILD_LIBMPG123
 pkgconfig_DATA += libmpg123.pc
-AM_CPPFLAGS += -I$(top_srcdir)/src/libmpg123
 endif
 if BUILD_LIBOUT123
 pkgconfig_DATA += libout123.pc
-AM_CPPFLAGS += -I$(top_srcdir)/src/libout123
 endif
 if BUILD_LIBSYN123
 pkgconfig_DATA += libsyn123.pc
-AM_CPPFLAGS += -I$(top_srcdir)/src/libsyn123
 endif
 endif
 
diff --git a/NEWS b/NEWS
index b9c2cb7d..70784969 100644
--- a/NEWS
+++ b/NEWS
@@ -1,7 +1,7 @@
 1.32.4
 ------
 - build:
--- Reorganize shared headers.
+-- Reorganize shared headers, API headers into src/include.
 -- Group C99 feature checks and make several standard headers
    mandatory.
 -- Get rid of SIZE_P, OFF_P and friends.
@@ -9,8 +9,9 @@
    to build individual modules using --disable-components logic.
 -- Allow building of programs with installed mpg123 libraries (--enable-programs,
    using pkg-config to locate libmpg123 et al., prompted by
-   https://github.com/madebr/mpg123/pull/11).
--- Only libmpg123 component installs fmt123.h.
+   https://github.com/madebr/mpg123/pull/11). Use with care (will fail on
+   version mismatch). Also handle that fmt123.h is installed identically
+   by each library component.
 - out123:
 -- added --libversion
 
diff --git a/configure.ac b/configure.ac
index bfd2d583..4e8e45b4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -32,7 +32,7 @@ m4_undefine([V_HEADER])
 
 AC_INIT([mpg123], V_MAJOR.V_MINOR.V_PATCH[]V_SUFFIX, [maintainer@mpg123.org])
 
-m4_define([V_HEADER], m4_esyscmd(grep -e _API_VERSION -e _PATCHLEVEL src/libmpg123/mpg123.h src/libout123/out123.h src/libsyn123/syn123.h))
+m4_define([V_HEADER], m4_esyscmd(grep -e _API_VERSION -e _PATCHLEVEL src/include/mpg123.h src/include/out123.h src/include/syn123.h))
 
 dnl libmpg123
 m4_define([API_VERSION], m4_bregexp(V_HEADER,
@@ -487,20 +487,21 @@ if test "x$build_programs" = xyes; then
   components="$components programs"
 fi
 
+use_extlib=no
 LIBMPG123_LD=abugifused
 if test "x$build_libmpg123" = xyes; then
   components="$components libmpg123"
 elif test "x$build_programs" = xyes; then
+  use_extlib=yes
   PKG_CHECK_MODULES(LIBMPG123_PKG, libmpg123, HAVE_LIBMPG123="yes", HAVE_LIBMPG123="no" missing_lib=yes)
-  CFLAGS="$CFLAGS $LIBMPG123_PKG_CFLAGS"
   LIBMPG123_LD="$LIBMPG123_PKG_LIBS"
 fi
 LIBOUT123_LD=abugifused
 if test "x$build_libout123" = xyes; then
   components="$components libout123"
 elif test "x$build_programs" = xyes; then
+  use_extlib=yes
   PKG_CHECK_MODULES(LIBOUT123_PKG, libout123, HAVE_LIBOUT123="yes", HAVE_LIBOUT123="no" missing_lib=yes)
-  CFLAGS="$CFLAGS $LIBOUT123_PKG_CFLAGS"
   LIBOUT123_LD="$LIBOUT123_PKG_LIBS"
 fi
 if test "x$build_libout123_modules" = xyes; then
@@ -510,8 +511,8 @@ LIBSYN123=abugifused
 if test "x$build_libsyn123" = xyes; then
   components="$components libsyn123"
 elif test "x$build_programs" = xyes; then
+  use_extlib=yes
   PKG_CHECK_MODULES(LIBSYN123_PKG, libsyn123, HAVE_LIBSYN123="yes", HAVE_LIBSYN123="no" missing_lib=yes)
-  CFLAGS="$CFLAGS $LIBSYN123_PKG_CFLAGS"
   LIBSYN123_LD="$LIBSYN123_PKG_LIBS"
 fi
 
@@ -3062,6 +3063,14 @@ if test x"$cpu_type" = xi486; then
   echo "only."
 fi
 
+if test "x$use_extlib" = xyes; then
+  echo
+  echo "Warning: You opted to use external builds of the mpg123 libraries. They need to"
+  echo "         match the headers shipped with this source (which are used during build"
+  echo "         regardless)."
+  echo
+fi
+
 echo
 echo "  CPPFLAGS='$CPPFLAGS'"
 echo "  CFLAGS='$CFLAGS'"
diff --git a/ports/cmake/src/CMakeLists.txt b/ports/cmake/src/CMakeLists.txt
index bc65c53e..09e44922 100644
--- a/ports/cmake/src/CMakeLists.txt
+++ b/ports/cmake/src/CMakeLists.txt
@@ -343,9 +343,7 @@ include_directories(
     "${CMAKE_CURRENT_BINARY_DIR}"
     "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/compat"
     "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/common"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/libsyn123"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/libout123"
-    "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/libmpg123")
+    "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/include")
 
 add_compile_definitions(
     $<$<BOOL:${WIN32}>:_CRT_SECURE_NO_WARNINGS>
diff --git a/ports/cmake/src/libmpg123/CMakeLists.txt b/ports/cmake/src/libmpg123/CMakeLists.txt
index d565867d..24271dc4 100644
--- a/ports/cmake/src/libmpg123/CMakeLists.txt
+++ b/ports/cmake/src/libmpg123/CMakeLists.txt
@@ -252,8 +252,8 @@ install(TARGETS ${TARGET} EXPORT targets
     ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}/"
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}/"
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}/")
-install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/libmpg123/mpg123.h"
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/include/mpg123.h"
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
-install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/fmt123.h"
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/include/fmt123.h"
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
 
diff --git a/ports/cmake/src/libout123/CMakeLists.txt b/ports/cmake/src/libout123/CMakeLists.txt
index cef878de..d0fe3586 100644
--- a/ports/cmake/src/libout123/CMakeLists.txt
+++ b/ports/cmake/src/libout123/CMakeLists.txt
@@ -31,5 +31,5 @@ install(TARGETS ${TARGET} EXPORT targets
     ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}/"
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}/"
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}/")
-install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/libout123/out123.h"
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/include/out123.h"
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
diff --git a/ports/cmake/src/libsyn123/CMakeLists.txt b/ports/cmake/src/libsyn123/CMakeLists.txt
index 6f9209a4..5c0ae0b4 100644
--- a/ports/cmake/src/libsyn123/CMakeLists.txt
+++ b/ports/cmake/src/libsyn123/CMakeLists.txt
@@ -26,5 +26,5 @@ install(TARGETS ${TARGET} EXPORT targets
     ARCHIVE DESTINATION "${CMAKE_INSTALL_LIBDIR}/"
     LIBRARY DESTINATION "${CMAKE_INSTALL_LIBDIR}/"
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}/")
-install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/libsyn123/syn123.h"
+install(FILES "${CMAKE_CURRENT_SOURCE_DIR}/../../../../src/include/syn123.h"
     DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}")
diff --git a/src/Makemodule.am b/src/Makemodule.am
index fbde8788..f8a8ab53 100644
--- a/src/Makemodule.am
+++ b/src/Makemodule.am
@@ -21,6 +21,13 @@ bin_PROGRAMS += \
 endif
 endif
 
+if NEED_LIB
+if NEED_MAINLIB
+# All main libraries need fmt123 header installed.
+include_HEADERS += src/include/fmt123.h
+endif
+endif
+
 src_mpg123_LDADD = \
   src/compat/libcompat.la \
   $(LIBMPG123) \
diff --git a/src/fmt123.h b/src/include/fmt123.h
similarity index 100%
rename from src/fmt123.h
rename to src/include/fmt123.h
diff --git a/src/libmpg123/mpg123.h b/src/include/mpg123.h
similarity index 100%
rename from src/libmpg123/mpg123.h
rename to src/include/mpg123.h
diff --git a/src/libout123/out123.h b/src/include/out123.h
similarity index 100%
rename from src/libout123/out123.h
rename to src/include/out123.h
diff --git a/src/libsyn123/syn123.h b/src/include/syn123.h
similarity index 100%
rename from src/libsyn123/syn123.h
rename to src/include/syn123.h
diff --git a/src/libmpg123/Makemodule.am b/src/libmpg123/Makemodule.am
index 00ae4788..5d28a123 100644
--- a/src/libmpg123/Makemodule.am
+++ b/src/libmpg123/Makemodule.am
@@ -20,10 +20,7 @@ src_libmpg123_calctables_LDADD = @LIBM@
 
 if BUILD_LIBMPG123
 lib_LTLIBRARIES += src/libmpg123/libmpg123.la
-include_HEADERS += src/libmpg123/mpg123.h
-# All libs need it, but only libmpg123 installs it.
-# Placing it in src/ helps other components to always find it.
-include_HEADERS += src/fmt123.h
+include_HEADERS += src/include/mpg123.h
 endif
 
 src_libmpg123_libmpg123_la_CFLAGS = @LIB_CFLAGS@
@@ -39,7 +36,6 @@ src_libmpg123_libmpg123_la_DEPENDENCIES = \
   src/compat/libcompat.la
 
 src_libmpg123_libmpg123_la_SOURCES = \
-  src/fmt123.h \
   src/libmpg123/mpeghead.h \
   src/libmpg123/parse.c \
   src/libmpg123/parse.h \
diff --git a/src/libout123/Makemodule.am b/src/libout123/Makemodule.am
index fcc5f87d..f8ce0260 100644
--- a/src/libout123/Makemodule.am
+++ b/src/libout123/Makemodule.am
@@ -18,7 +18,7 @@ if !HAVE_MODULES
 include src/libout123/modules/Makemodule.am
 endif
 
-include_HEADERS += src/libout123/out123.h
+include_HEADERS += src/include/out123.h
 
 noinst_LTLIBRARIES += src/libout123/libmodule.la
 
diff --git a/src/libsyn123/Makemodule.am b/src/libsyn123/Makemodule.am
index d85824fa..5a4f3ae3 100644
--- a/src/libsyn123/Makemodule.am
+++ b/src/libsyn123/Makemodule.am
@@ -4,7 +4,7 @@
 # and format conversion for the mpg123 project.
 
 if BUILD_LIBSYN123
-include_HEADERS += src/libsyn123/syn123.h
+include_HEADERS += src/include/syn123.h
 
 lib_LTLIBRARIES += src/libsyn123/libsyn123.la
 endif
diff --git a/src/tests/noise.c b/src/tests/noise.c
index 4af8cf6b..cdbfcc9b 100644
--- a/src/tests/noise.c
+++ b/src/tests/noise.c
@@ -1,7 +1,7 @@
 #include "config.h"
-#include "compat.h"
-#include "dither.h"
-#include "debug.h"
+#include "../compat/compat.h"
+#include "../libmpg123/dither.h"
+#include "../common/debug.h"
 
 /* Directly include the code for testing, avoiding
    build of same object with and without libtool. */