From 1956696da519ce1b2fca4dbc04da8ddf37892945 Mon Sep 17 00:00:00 2001 From: "Chrissi^ (Chris Fiege)" Date: Thu, 4 Jun 2020 22:35:58 +0200 Subject: [PATCH] openwrt: Store Kernel Debug-Info (#1971) This change stores a Kernel with Debug-Symbols for the current architecture in a new output directory '/debug'. This allows a developer or operator of a network to store the kernel along with the actual images. In case of a kernel oops the debug information can be used with the script 'scripts/decode_stacktrace.sh' in the kernel source tree to get the names to the symbols of the stack trace. OpenWRT already provides the CONFIG_COLLECT_KERNEL_DEBUG -option that creates a kernel with debug-symbols in the OpenWRT output directory. This change enables this option and copies the generated kernel to the gluon output directory. Signed-off-by: Chrissi^ --- Makefile | 3 ++- docs/dev/debugging.rst | 51 +++++++++++++++++++++++++++++++++++ docs/index.rst | 1 + docs/user/getting_started.rst | 6 +++++ scripts/copy_output.lua | 15 +++++++++++ targets/generic | 2 ++ 6 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 docs/dev/debugging.rst diff --git a/Makefile b/Makefile index b220afe9..e121d3c8 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ GLUON_TMPDIR ?= tmp GLUON_OUTPUTDIR ?= output GLUON_IMAGEDIR ?= $(GLUON_OUTPUTDIR)/images GLUON_PACKAGEDIR ?= $(GLUON_OUTPUTDIR)/packages +GLUON_DEBUGDIR ?= $(GLUON_OUTPUTDIR)/debug GLUON_TARGETSDIR ?= targets GLUON_PATCHESDIR ?= patches @@ -56,7 +57,7 @@ endef GLUON_VARS = \ GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_AUTOREMOVE GLUON_DEBUG GLUON_MINIFY GLUON_DEPRECATED \ - GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR \ + GLUON_DEVICES GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR GLUON_DEBUGDIR \ GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS \ GLUON_TARGET BOARD SUBTARGET diff --git a/docs/dev/debugging.rst b/docs/dev/debugging.rst new file mode 100644 index 00000000..413f1c97 --- /dev/null +++ b/docs/dev/debugging.rst @@ -0,0 +1,51 @@ +Debugging +========= + + +.. _dev-debugging-kernel-oops: + +Kernel Oops +----------- + +Sometimes a running Linux kernel detects an error during runtime that canot +be corrected. +This usually generates a stack trace that points to the location in the code +that caused the oops. + +Linux kernels in Gluon (and OpenWrt) are stripped. +That means they do not contain any debug symbols. +On one hand this leads to a smaller binary and faster loading times on the +target. +On the other hand this means that in a case of a stack trace the unwinder +can only print memory locations and no further debugging information. + +Gluon stores a compressed kernel with debug symbols for every target +in the directory `output/debug/`. +These kernels should be kept along with the images as long as the images +are in use. +This allows the developer to analyse a stack trace later. + +Decoding Stacktraces +.................... + +The tooling is contained in the kernel source tree in the file +`decode_stacktrace.sh `__. +This file and the needed source tree are available in the directory: :: + + openwrt/build_dir/target-/linux-/linux-/ + +.. note:: + Make sure to use a kernel tree that matches the version and patches + that was used to build the kernel. + If in doubt just re-build the images for the target. + +Some more information on how to use this tool can be found at +`LWN `__. + +Obtaining Stacktraces +..................... + +On many targets stacktraces can be read from the following +location after reboot: :: + + /sys/kernel/debug/crashlog diff --git a/docs/index.rst b/docs/index.rst index 9abd8dc5..cf767073 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -43,6 +43,7 @@ Several Freifunk communities in Germany use Gluon as the foundation of their Fre dev/mac_addresses dev/site_library dev/build + dev/debugging .. toctree:: :caption: gluon-web Reference diff --git a/docs/user/getting_started.rst b/docs/user/getting_started.rst index db642baf..36d9260e 100644 --- a/docs/user/getting_started.rst +++ b/docs/user/getting_started.rst @@ -113,6 +113,12 @@ symlinks, you can use the following command to resolve these links while copying cp -rL output/images /var/www +The directory `output/debug` contains a compressed kernel image for each +architecture. +These can be used for debugging and should be stored along with the images to +allow debugging of kernel problems on devices in the field. +See :ref:`Debugging ` for more information. + Cleaning the build tree ....................... diff --git a/scripts/copy_output.lua b/scripts/copy_output.lua index 91547964..9f14fba5 100755 --- a/scripts/copy_output.lua +++ b/scripts/copy_output.lua @@ -27,6 +27,7 @@ end mkdir(env.GLUON_IMAGEDIR..'/factory') mkdir(env.GLUON_IMAGEDIR..'/sysupgrade') mkdir(env.GLUON_IMAGEDIR..'/other') +mkdir(env.GLUON_DEBUGDIR) lib.include(target) @@ -66,6 +67,20 @@ for _, images in pairs(lib.images) do end end +-- copy kernel image with debug symbols +local kernel_debug_glob = string.format('%s/gluon-\0-%s-kernel-debug.tar.zst', + env.GLUON_DEBUGDIR, + target) +lib.exec {'rm', '-f', kernel_debug_glob} +local kernel_debug_source = string.format('openwrt/bin/targets/%s/kernel-debug.tar.zst', + bindir) +local kernel_debug_dest = string.format('%s/gluon-%s-%s-%s-kernel-debug.tar.zst', + env.GLUON_DEBUGDIR, + lib.site_code, + env.GLUON_RELEASE, + target) +lib.exec {'cp', kernel_debug_source, kernel_debug_dest} + -- Copy opkg repo if lib.opkg and (env.GLUON_DEVICES or '') == '' then diff --git a/targets/generic b/targets/generic index e0285821..c7f28452 100644 --- a/targets/generic +++ b/targets/generic @@ -47,6 +47,8 @@ try_config('TARGET_SQUASHFS_BLOCK_SIZE', 256) config('KERNEL_IP_MROUTE', false) config('KERNEL_IPV6_MROUTE', false) +config('COLLECT_KERNEL_DEBUG', true) + try_config('TARGET_MULTI_PROFILE', true) try_config('TARGET_PER_DEVICE_ROOTFS', true)