openwrt: Store Kernel Debug-Info (#1971)

This change stores a Kernel with Debug-Symbols for the current
architecture in a new output directory '<outputdir>/debug'.
This allows a developer or operator of a network to store the kernel
along with the actual images. In case of a kernel oops the debug
information can be used with the script
'scripts/decode_stacktrace.sh' in the kernel source tree to get the
names to the symbols of the stack trace.

OpenWRT already provides the CONFIG_COLLECT_KERNEL_DEBUG -option that
creates a kernel with debug-symbols in the OpenWRT output directory.
This change enables this option and copies the generated kernel to the
gluon output directory.

Signed-off-by: Chrissi^ <chris@tinyhost.de>

Makefile

@@ -33,6 +33,7 @@ GLUON_TMPDIR ?= tmp
 GLUON_OUTPUTDIR ?= output
 GLUON_IMAGEDIR ?= $(GLUON_OUTPUTDIR)/images
 GLUON_PACKAGEDIR ?= $(GLUON_OUTPUTDIR)/packages
+GLUON_DEBUGDIR ?= $(GLUON_OUTPUTDIR)/debug
 GLUON_TARGETSDIR ?= targets
 GLUON_PATCHESDIR ?= patches
 
@@ -56,7 +57,7 @@ endef
 
 GLUON_VARS = \
 	GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_AUTOREMOVE GLUON_DEBUG GLUON_MINIFY GLUON_DEPRECATED \
-	GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR \
+	GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR GLUON_DEBUGDIR \
 	GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS \
 	GLUON_TARGET BOARD SUBTARGET
 

docs/dev/debugging.rst

@@ -0,0 +1,51 @@
+Debugging
+=========
+
+
+.. _dev-debugging-kernel-oops:
+
+Kernel Oops
+-----------
+
+Sometimes a running Linux kernel detects an error during runtime that canot
+be corrected.
+This usually generates a stack trace that points to the location in the code
+that caused the oops.
+
+Linux kernels in Gluon (and OpenWrt) are stripped.
+That means they do not contain any debug symbols.
+On one hand this leads to a smaller binary and faster loading times on the
+target.
+On the other hand this means that in a case of a stack trace the unwinder
+can only print memory locations and no further debugging information.
+
+Gluon stores a compressed kernel with debug symbols for every target
+in the directory `output/debug/`.
+These kernels should be kept along with the images as long as the images
+are in use.
+This allows the developer to analyse a stack trace later.
+
+Decoding Stacktraces
+....................
+
+The tooling is contained in the kernel source tree in the file
+`decode_stacktrace.sh <https://github.com/torvalds/linux/blob/master/scripts/decode_stacktrace.sh>`__.
+This file and the needed source tree are available in the directory: ::
+
+    openwrt/build_dir/target-<architecture>/linux-<architecture>/linux-<version>/
+
+.. note::
+   Make sure to use a kernel tree that matches the version and patches
+   that was used to build the kernel.
+   If in doubt just re-build the images for the target.
+
+Some more information on how to use this tool can be found at
+`LWN <https://lwn.net/Articles/592724/>`__.
+
+Obtaining Stacktraces
+.....................
+
+On many targets stacktraces can be read from the following
+location after reboot: ::
+
+    /sys/kernel/debug/crashlog

docs/index.rst

@@ -43,6 +43,7 @@ Several Freifunk communities in Germany use Gluon as the foundation of their Fre
    dev/mac_addresses
    dev/site_library
    dev/build
+   dev/debugging
 
 .. toctree::
    :caption: gluon-web Reference

docs/user/getting_started.rst

@@ -113,6 +113,12 @@ symlinks, you can use the following command to resolve these links while copying
 
     cp -rL output/images /var/www
 
+The directory `output/debug` contains a compressed kernel image for each
+architecture.
+These can be used for debugging and should be stored along with the images to
+allow debugging of kernel problems on devices in the field.
+See :ref:`Debugging <dev-debugging-kernel-oops>` for more information.
+
 Cleaning the build tree
 .......................
 

scripts/copy_output.lua

@@ -27,6 +27,7 @@ end
 mkdir(env.GLUON_IMAGEDIR..'/factory')
 mkdir(env.GLUON_IMAGEDIR..'/sysupgrade')
 mkdir(env.GLUON_IMAGEDIR..'/other')
+mkdir(env.GLUON_DEBUGDIR)
 
 
 lib.include(target)
@@ -66,6 +67,20 @@ for _, images in pairs(lib.images) do
 	end
 end
 
+-- copy kernel image with debug symbols
+local kernel_debug_glob = string.format('%s/gluon-\0-%s-kernel-debug.tar.zst',
+        env.GLUON_DEBUGDIR,
+        target)
+lib.exec {'rm', '-f', kernel_debug_glob}
+local kernel_debug_source = string.format('openwrt/bin/targets/%s/kernel-debug.tar.zst',
+        bindir)
+local kernel_debug_dest = string.format('%s/gluon-%s-%s-%s-kernel-debug.tar.zst',
+        env.GLUON_DEBUGDIR,
+        lib.site_code,
+        env.GLUON_RELEASE,
+        target)
+lib.exec {'cp', kernel_debug_source, kernel_debug_dest}
+
 
 -- Copy opkg repo
 if lib.opkg and (env.GLUON_DEVICES or '') == '' then

targets/generic

@@ -47,6 +47,8 @@ try_config('TARGET_SQUASHFS_BLOCK_SIZE', 256)
 config('KERNEL_IP_MROUTE', false)
 config('KERNEL_IPV6_MROUTE', false)
 
+config('COLLECT_KERNEL_DEBUG', true)
+
 try_config('TARGET_MULTI_PROFILE', true)
 try_config('TARGET_PER_DEVICE_ROOTFS', true)