From d4dee692c671090c10dca8cd1e282575a18469cb Mon Sep 17 00:00:00 2001 From: lemoer Date: Fri, 23 Jul 2021 19:50:15 +0200 Subject: [PATCH] contrib: add push_pkg.sh to enhance package development workflow (v2) (#2023) v2: In contrast to the last patches, this is now built on top of ssh only, without using e.g. 9pfs. Furthermore it works also with arbitary remote hosts on any target/architecture. Also the scripts were renamed and moved to /scripts. The aim of this commit is to allow fast rebuild cycles during the development of gluon packages. Currently the following workflow can be used: # start a local qemu instance scripts/run_qemu.sh output/images/factory/[...].img # do your changes in the file you want to patch vi package/gluon-ebtables/files/etc/init.d/gluon-ebtables # rebuild and update the package scripts/push_pkg.sh package/gluon-ebtables/ # test your changes ... # do more changes ... # rebuild and update the package scripts/push_pkg.sh package/gluon-ebtables/ # test your changes ... (and so on...) Implementation details: - Currently this is based on ssh/scp. - Opkg is used to install/update the packages in the remote machine. Benefits: - This works with compiled and non-compiled packages. - This works with native OpenWrt and Gluon packages. - This even performs the check_site.lua checks as they are integrated as post_install scripts into the openwrt package. - It works for all architectures/targets. --- contrib/push_pkg.sh | 149 ++++++++++++++++++++++++++++++++++++++++++ contrib/run_qemu.sh | 15 +++++ docs/dev/packages.rst | 79 ++++++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100755 contrib/push_pkg.sh create mode 100755 contrib/run_qemu.sh diff --git a/contrib/push_pkg.sh b/contrib/push_pkg.sh new file mode 100755 index 00000000..4b9fe740 --- /dev/null +++ b/contrib/push_pkg.sh @@ -0,0 +1,149 @@ +#!/bin/sh + +set -e + +topdir="$(realpath "$(dirname "${0}")/../openwrt")" + +# defaults to qemu run script +ssh_host=localhost +build_only=0 +preserve_config=1 + +print_help() { + echo "$0 [OPTIONS] PACAKGE_DIR [PACKAGE_DIR] ..." + echo "" + echo " -h print this help" + echo " -r HOST use a remote machine as target machine. By default if this" + echo " option is not given, push_pkg.sh will use a locally" + echo " running qemu instance started by run_qemu.sh." + echo " -p PORT use PORT as ssh port (default is 22)" + echo " -b build only, do not push" + echo " -P do not preserve /etc/config. By default, if a package" + echo " defines a config file in /etc/config, this config file" + echo " will be preserved. If you specify this flag, the package" + echo " default will be installed instead." + echo "" + echo ' To change gluon variables, run e.g. "make config GLUON_MINIFY=0"' + echo ' because then the gluon logic will be triggered, and openwrt/.config' + echo ' will be regenerated. The variables from openwrt/.config are already' + echo ' automatically used for this script.' + echo +} + +while getopts "p:r:hbP" opt +do + case $opt in + P) preserve_config=0;; + p) ssh_port="${OPTARG}";; + r) ssh_host="${OPTARG}"; [ -z "$ssh_port" ] && ssh_port=22;; + b) build_only=1;; + h) print_help; exit 0;; + *) ;; + esac +done +shift $(( OPTIND - 1 )) + +[ -z "$ssh_port" ] && ssh_port=2223 + +if [ "$build_only" -eq 0 ]; then + remote_info=$(ssh -p "${ssh_port}" "root@${ssh_host}" ' + source /etc/os-release + printf "%s\t%s\n" "$OPENWRT_BOARD" "$OPENWRT_ARCH" + ') + REMOTE_OPENWRT_BOARD="$(echo "$remote_info" | cut -f 1)" + REMOTE_OPENWRT_ARCH="$(echo "$remote_info" | cut -f 2)" + + # check target + if ! grep -q "CONFIG_TARGET_ARCH_PACKAGES=\"${REMOTE_OPENWRT_ARCH}\"" "${topdir}/.config"; then + echo "Configured OpenWrt Target is not matching with the target machine!" 1>&2 + echo + printf "%s" " Configured architecture: " 1>&2 + grep "CONFIG_TARGET_ARCH_PACKAGES" "${topdir}/.config" 1>&2 + echo "Target machine architecture: ${REMOTE_OPENWRT_ARCH}" 1>&2 + echo 1>&2 + echo "To switch the local with the run with the corresponding GLUON_TARGET:" 1>&2 + echo " make GLUON_TARGET=... config" 1>&2 + exit 1 + fi +fi + +if [ $# -lt 1 ]; then + echo ERROR: Please specify a PACKAGE_DIR. For example: + echo + echo " \$ $0 package/gluon-core" + exit 1 +fi + +while [ $# -gt 0 ]; do + + pkgdir="$1"; shift + echo "Package: ${pkgdir}" + + if ! [ -f "${pkgdir}/Makefile" ]; then + echo "ERROR: ${pkgdir} does not contain a Makefile" + exit 1 + fi + + if ! grep -q BuildPackage "${pkgdir}/Makefile"; then + echo "ERROR: ${pkgdir}/Makefile does not contain a BuildPackage command" + exit 1 + fi + + opkg_packages="$(make TOPDIR="${topdir}" -C "${pkgdir}" DUMP=1 | awk '/^Package: / { print $2 }')" + + search_package() { + find "$2" -name "$1_*.ipk" -printf "%f\n" + } + + make TOPDIR="${topdir}" -C "${pkgdir}" clean + make TOPDIR="${topdir}" -C "${pkgdir}" compile + + if [ "$build_only" -eq 1 ]; then + continue + fi + + # IPv6 addresses need brackets around the ${ssh_host} for scp! + if echo "${ssh_host}" | grep -q :; then + BL=[ + BR=] + fi + + for pkg in ${opkg_packages}; do + + for feed in "${topdir}/bin/packages/${REMOTE_OPENWRT_ARCH}/"*/ "${topdir}/bin/targets/${REMOTE_OPENWRT_BOARD}/packages/"; do + printf "%s" "searching ${pkg} in ${feed}: " + filename=$(search_package "${pkg}" "${feed}") + if [ -n "${filename}" ]; then + echo found! + break + else + echo not found + fi + done + + if [ "$preserve_config" -eq 0 ]; then + opkg_flags=" --force-maintainer" + fi + + # shellcheck disable=SC2029 + if [ -n "$filename" ]; then + scp -P "${ssh_port}" "$feed/$filename" "root@${BL}${ssh_host}${BR}:/tmp/${filename}" + ssh -p "${ssh_port}" "root@${ssh_host}" " + set -e + echo Running opkg: + opkg install --force-reinstall ${opkg_flags} '/tmp/${filename}' + rm '/tmp/${filename}' + gluon-reconfigure + " + else + # Some packages (e.g. procd-seccomp) seem to contain BuildPackage commands + # which do not generate *.ipk files. Till this point, I am not aware why + # this is happening. However, dropping a warning if the corresponding + # *.ipk is not found (maybe due to other reasons as well), seems to + # be more reasonable than aborting. Before this commit, the command + # has failed. + echo "Warning: ${pkg}*.ipk not found! Ignoring." 1>&2 + fi + + done +done diff --git a/contrib/run_qemu.sh b/contrib/run_qemu.sh new file mode 100755 index 00000000..3166f7e0 --- /dev/null +++ b/contrib/run_qemu.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Note: You can exit the qemu instance by first pressing "CTRL + a" then "c". +# Then you enter the command mode of qemu and can exit by typing "quit". + +qemu-system-x86_64 \ + -d 'cpu_reset' \ + -enable-kvm \ + -gdb tcp::1234 \ + -nographic \ + -netdev user,id=wan,hostfwd=tcp::2223-10.0.2.15:22 \ + -device virtio-net-pci,netdev=wan,addr=0x06,id=nic1 \ + -netdev user,id=lan,hostfwd=tcp::6080-192.168.1.1:80,hostfwd=tcp::2222-192.168.1.1:22,net=192.168.1.100/24 \ + -device virtio-net-pci,netdev=lan,addr=0x05,id=nic2 \ + "$@" diff --git a/docs/dev/packages.rst b/docs/dev/packages.rst index b6032b2e..098b0276 100644 --- a/docs/dev/packages.rst +++ b/docs/dev/packages.rst @@ -3,6 +3,85 @@ Package development Gluon packages are OpenWrt packages and follow the same rules described at https://openwrt.org/docs/guide-developer/packages. +Development workflow +==================== + +When you are developing packages, it often happens that you iteratively want to deploy +and verify the state your development. There are two ways to verify your changes: + +1) One way is to rebuild the complete firmware, flash it, configure it and verify your + development then. This usually takes at least a few minutes to get your changes + working so you can test them. Especially if you iterate a lot, this becomes tedious. +2) Another way is to rebuild only the package you are currently working on and + to deploy this package to your test system. Here not even a reboot is required. + This makes iterating relatively fast. Your test system could be real hardware or + even a qemu in most cases. + +Gluon provides scripts to enhance workflow 2). Here is an example illustrating +the workflow using these scripts: + +.. code-block:: shell + + # start a local qemu instance + contrib/run_qemu.sh output/images/factory/[...]-x86-64.img + + # apply changes to the desired package + vi package/gluon-ebtables/files/etc/init.d/gluon-ebtables + + # rebuild and push the package to the qemu instance + contrib/push_pkg.sh package/gluon-ebtables/ + + # test your changes + ... + + # do more changes + ... + + # rebuild and push the package to the qemu instance + contrib/push_pkg.sh package/gluon-ebtables/ + + # test your changes + ... + + (and so on...) + + # see help of the script for more information + contrib/push_pkg.sh -h + ... + +Features of ``push_pkg.sh``: + +* Works with compiled and non-compiled packages. + + * This means it can be used in the development of C-code, Lua-Code and mostly any other code. + +* Works with native OpenWrt and Gluon packages. +* Pushes to remote machines or local qemu instances. +* Pushes multiple packages in in one call if desired. +* Performs site.conf checks. + +Implementation details of ``push_pkg.sh``: + +* First, the script builds an opkg package using the OpenWrt build system. +* This package is pushed to a *target machine* using scp: + + * By default the *target machine* is a locally running x86 qemu started using ``run_qemu.sh``. + * The *target machine* can also be remote machine. (See the cli switch ``-r``) + * Remote machines are not limited to a specific architecture. All architectures supported by gluon can be used as remote machines. + +* Finally opkg is used to install/update the packages in the target machine. + + * While doing this, it will not override ``/etc/config`` with package defaults by default. (See the cli switch ``-P``). + * While doing this, opkg calls the ``check_site.lua`` from the package as post_install script to validate the ``site.conf``. This means that the ``site.conf`` of the target machine is used for this validation. + +Note that: + +* ``push_pkg.sh`` does neither build nor push dependencies of the packages automatically. If you want to update dependencies, you must explicitly specify them to be pushed. +* If you add new packages, you must run ``make update config GLUON_TARGET=...``. +* You can change the gluon target of the target machine via ``make config GLUON_TARGET=...``. +* If you want to update the ``site.conf`` of the target machine, use ``push_pkg.sh package/gluon-site/``. +* Sometimes when things break, you can heal them by compiling a package with its dependencies: ``cd openwrt; make package/gluon-ebtables/clean; make package/gluon-ebtables/compile; cd ..``. +* You can exit qemu by pressing ``CTRL + a`` and ``c`` afterwards. Gluon package makefiles =======================