contrib: add push_pkg.sh to enhance package development workflow (v2) (#2023)
v2: In contrast to the last patches, this is now built on top of ssh
only, without using e.g. 9pfs. Furthermore it works also with
arbitary remote hosts on any target/architecture. Also the
scripts were renamed and moved to /scripts.
The aim of this commit is to allow fast rebuild cycles during the
development of gluon packages.
Currently the following workflow can be used:
# start a local qemu instance
scripts/run_qemu.sh output/images/factory/[...].img
# do your changes in the file you want to patch
vi package/gluon-ebtables/files/etc/init.d/gluon-ebtables
# rebuild and update the package
scripts/push_pkg.sh package/gluon-ebtables/
# test your changes
...
# do more changes
...
# rebuild and update the package
scripts/push_pkg.sh package/gluon-ebtables/
# test your changes
...
(and so on...)
Implementation details:
- Currently this is based on ssh/scp.
- Opkg is used to install/update the packages in the remote machine.
Benefits:
- This works with compiled and non-compiled packages.
- This works with native OpenWrt and Gluon packages.
- This even performs the check_site.lua checks as they are integrated
as post_install scripts into the openwrt package.
- It works for all architectures/targets.
This commit is contained in:
lemoer
GitHub
parent
f4a3afe8fb
commit
d4dee692c6
149
contrib/push_pkg.sh
Executable file
149
contrib/push_pkg.sh
Executable file
@ -0,0 +1,149 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
topdir="$(realpath "$(dirname "${0}")/../openwrt")"
|
||||||
|
|
||||||
|
# defaults to qemu run script
|
||||||
|
ssh_host=localhost
|
||||||
|
build_only=0
|
||||||
|
preserve_config=1
|
||||||
|
|
||||||
|
print_help() {
|
||||||
|
echo "$0 [OPTIONS] PACAKGE_DIR [PACKAGE_DIR] ..."
|
||||||
|
echo ""
|
||||||
|
echo " -h print this help"
|
||||||
|
echo " -r HOST use a remote machine as target machine. By default if this"
|
||||||
|
echo " option is not given, push_pkg.sh will use a locally"
|
||||||
|
echo " running qemu instance started by run_qemu.sh."
|
||||||
|
echo " -p PORT use PORT as ssh port (default is 22)"
|
||||||
|
echo " -b build only, do not push"
|
||||||
|
echo " -P do not preserve /etc/config. By default, if a package"
|
||||||
|
echo " defines a config file in /etc/config, this config file"
|
||||||
|
echo " will be preserved. If you specify this flag, the package"
|
||||||
|
echo " default will be installed instead."
|
||||||
|
echo ""
|
||||||
|
echo ' To change gluon variables, run e.g. "make config GLUON_MINIFY=0"'
|
||||||
|
echo ' because then the gluon logic will be triggered, and openwrt/.config'
|
||||||
|
echo ' will be regenerated. The variables from openwrt/.config are already'
|
||||||
|
echo ' automatically used for this script.'
|
||||||
|
echo
|
||||||
|
}
|
||||||
|
|
||||||
|
while getopts "p:r:hbP" opt
|
||||||
|
do
|
||||||
|
case $opt in
|
||||||
|
P) preserve_config=0;;
|
||||||
|
p) ssh_port="${OPTARG}";;
|
||||||
|
r) ssh_host="${OPTARG}"; [ -z "$ssh_port" ] && ssh_port=22;;
|
||||||
|
b) build_only=1;;
|
||||||
|
h) print_help; exit 0;;
|
||||||
|
*) ;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
shift $(( OPTIND - 1 ))
|
||||||
|
|
||||||
|
[ -z "$ssh_port" ] && ssh_port=2223
|
||||||
|
|
||||||
|
if [ "$build_only" -eq 0 ]; then
|
||||||
|
remote_info=$(ssh -p "${ssh_port}" "root@${ssh_host}" '
|
||||||
|
source /etc/os-release
|
||||||
|
printf "%s\t%s\n" "$OPENWRT_BOARD" "$OPENWRT_ARCH"
|
||||||
|
')
|
||||||
|
REMOTE_OPENWRT_BOARD="$(echo "$remote_info" | cut -f 1)"
|
||||||
|
REMOTE_OPENWRT_ARCH="$(echo "$remote_info" | cut -f 2)"
|
||||||
|
|
||||||
|
# check target
|
||||||
|
if ! grep -q "CONFIG_TARGET_ARCH_PACKAGES=\"${REMOTE_OPENWRT_ARCH}\"" "${topdir}/.config"; then
|
||||||
|
echo "Configured OpenWrt Target is not matching with the target machine!" 1>&2
|
||||||
|
echo
|
||||||
|
printf "%s" " Configured architecture: " 1>&2
|
||||||
|
grep "CONFIG_TARGET_ARCH_PACKAGES" "${topdir}/.config" 1>&2
|
||||||
|
echo "Target machine architecture: ${REMOTE_OPENWRT_ARCH}" 1>&2
|
||||||
|
echo 1>&2
|
||||||
|
echo "To switch the local with the run with the corresponding GLUON_TARGET:" 1>&2
|
||||||
|
echo " make GLUON_TARGET=... config" 1>&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $# -lt 1 ]; then
|
||||||
|
echo ERROR: Please specify a PACKAGE_DIR. For example:
|
||||||
|
echo
|
||||||
|
echo " \$ $0 package/gluon-core"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
while [ $# -gt 0 ]; do
|
||||||
|
|
||||||
|
pkgdir="$1"; shift
|
||||||
|
echo "Package: ${pkgdir}"
|
||||||
|
|
||||||
|
if ! [ -f "${pkgdir}/Makefile" ]; then
|
||||||
|
echo "ERROR: ${pkgdir} does not contain a Makefile"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! grep -q BuildPackage "${pkgdir}/Makefile"; then
|
||||||
|
echo "ERROR: ${pkgdir}/Makefile does not contain a BuildPackage command"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
opkg_packages="$(make TOPDIR="${topdir}" -C "${pkgdir}" DUMP=1 | awk '/^Package: / { print $2 }')"
|
||||||
|
|
||||||
|
search_package() {
|
||||||
|
find "$2" -name "$1_*.ipk" -printf "%f\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
make TOPDIR="${topdir}" -C "${pkgdir}" clean
|
||||||
|
make TOPDIR="${topdir}" -C "${pkgdir}" compile
|
||||||
|
|
||||||
|
if [ "$build_only" -eq 1 ]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# IPv6 addresses need brackets around the ${ssh_host} for scp!
|
||||||
|
if echo "${ssh_host}" | grep -q :; then
|
||||||
|
BL=[
|
||||||
|
BR=]
|
||||||
|
fi
|
||||||
|
|
||||||
|
for pkg in ${opkg_packages}; do
|
||||||
|
|
||||||
|
for feed in "${topdir}/bin/packages/${REMOTE_OPENWRT_ARCH}/"*/ "${topdir}/bin/targets/${REMOTE_OPENWRT_BOARD}/packages/"; do
|
||||||
|
printf "%s" "searching ${pkg} in ${feed}: "
|
||||||
|
filename=$(search_package "${pkg}" "${feed}")
|
||||||
|
if [ -n "${filename}" ]; then
|
||||||
|
echo found!
|
||||||
|
break
|
||||||
|
else
|
||||||
|
echo not found
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
if [ "$preserve_config" -eq 0 ]; then
|
||||||
|
opkg_flags=" --force-maintainer"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# shellcheck disable=SC2029
|
||||||
|
if [ -n "$filename" ]; then
|
||||||
|
scp -P "${ssh_port}" "$feed/$filename" "root@${BL}${ssh_host}${BR}:/tmp/${filename}"
|
||||||
|
ssh -p "${ssh_port}" "root@${ssh_host}" "
|
||||||
|
set -e
|
||||||
|
echo Running opkg:
|
||||||
|
opkg install --force-reinstall ${opkg_flags} '/tmp/${filename}'
|
||||||
|
rm '/tmp/${filename}'
|
||||||
|
gluon-reconfigure
|
||||||
|
"
|
||||||
|
else
|
||||||
|
# Some packages (e.g. procd-seccomp) seem to contain BuildPackage commands
|
||||||
|
# which do not generate *.ipk files. Till this point, I am not aware why
|
||||||
|
# this is happening. However, dropping a warning if the corresponding
|
||||||
|
# *.ipk is not found (maybe due to other reasons as well), seems to
|
||||||
|
# be more reasonable than aborting. Before this commit, the command
|
||||||
|
# has failed.
|
||||||
|
echo "Warning: ${pkg}*.ipk not found! Ignoring." 1>&2
|
||||||
|
fi
|
||||||
|
|
||||||
|
done
|
||||||
|
done
|
||||||
15
contrib/run_qemu.sh
Executable file
15
contrib/run_qemu.sh
Executable file
@ -0,0 +1,15 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Note: You can exit the qemu instance by first pressing "CTRL + a" then "c".
|
||||||
|
# Then you enter the command mode of qemu and can exit by typing "quit".
|
||||||
|
|
||||||
|
qemu-system-x86_64 \
|
||||||
|
-d 'cpu_reset' \
|
||||||
|
-enable-kvm \
|
||||||
|
-gdb tcp::1234 \
|
||||||
|
-nographic \
|
||||||
|
-netdev user,id=wan,hostfwd=tcp::2223-10.0.2.15:22 \
|
||||||
|
-device virtio-net-pci,netdev=wan,addr=0x06,id=nic1 \
|
||||||
|
-netdev user,id=lan,hostfwd=tcp::6080-192.168.1.1:80,hostfwd=tcp::2222-192.168.1.1:22,net=192.168.1.100/24 \
|
||||||
|
-device virtio-net-pci,netdev=lan,addr=0x05,id=nic2 \
|
||||||
|
"$@"
|
||||||
@ -3,6 +3,85 @@ Package development
|
|||||||
|
|
||||||
Gluon packages are OpenWrt packages and follow the same rules described at https://openwrt.org/docs/guide-developer/packages.
|
Gluon packages are OpenWrt packages and follow the same rules described at https://openwrt.org/docs/guide-developer/packages.
|
||||||
|
|
||||||
|
Development workflow
|
||||||
|
====================
|
||||||
|
|
||||||
|
When you are developing packages, it often happens that you iteratively want to deploy
|
||||||
|
and verify the state your development. There are two ways to verify your changes:
|
||||||
|
|
||||||
|
1) One way is to rebuild the complete firmware, flash it, configure it and verify your
|
||||||
|
development then. This usually takes at least a few minutes to get your changes
|
||||||
|
working so you can test them. Especially if you iterate a lot, this becomes tedious.
|
||||||
|
2) Another way is to rebuild only the package you are currently working on and
|
||||||
|
to deploy this package to your test system. Here not even a reboot is required.
|
||||||
|
This makes iterating relatively fast. Your test system could be real hardware or
|
||||||
|
even a qemu in most cases.
|
||||||
|
|
||||||
|
Gluon provides scripts to enhance workflow 2). Here is an example illustrating
|
||||||
|
the workflow using these scripts:
|
||||||
|
|
||||||
|
.. code-block:: shell
|
||||||
|
|
||||||
|
# start a local qemu instance
|
||||||
|
contrib/run_qemu.sh output/images/factory/[...]-x86-64.img
|
||||||
|
|
||||||
|
# apply changes to the desired package
|
||||||
|
vi package/gluon-ebtables/files/etc/init.d/gluon-ebtables
|
||||||
|
|
||||||
|
# rebuild and push the package to the qemu instance
|
||||||
|
contrib/push_pkg.sh package/gluon-ebtables/
|
||||||
|
|
||||||
|
# test your changes
|
||||||
|
...
|
||||||
|
|
||||||
|
# do more changes
|
||||||
|
...
|
||||||
|
|
||||||
|
# rebuild and push the package to the qemu instance
|
||||||
|
contrib/push_pkg.sh package/gluon-ebtables/
|
||||||
|
|
||||||
|
# test your changes
|
||||||
|
...
|
||||||
|
|
||||||
|
(and so on...)
|
||||||
|
|
||||||
|
# see help of the script for more information
|
||||||
|
contrib/push_pkg.sh -h
|
||||||
|
...
|
||||||
|
|
||||||
|
Features of ``push_pkg.sh``:
|
||||||
|
|
||||||
|
* Works with compiled and non-compiled packages.
|
||||||
|
|
||||||
|
* This means it can be used in the development of C-code, Lua-Code and mostly any other code.
|
||||||
|
|
||||||
|
* Works with native OpenWrt and Gluon packages.
|
||||||
|
* Pushes to remote machines or local qemu instances.
|
||||||
|
* Pushes multiple packages in in one call if desired.
|
||||||
|
* Performs site.conf checks.
|
||||||
|
|
||||||
|
Implementation details of ``push_pkg.sh``:
|
||||||
|
|
||||||
|
* First, the script builds an opkg package using the OpenWrt build system.
|
||||||
|
* This package is pushed to a *target machine* using scp:
|
||||||
|
|
||||||
|
* By default the *target machine* is a locally running x86 qemu started using ``run_qemu.sh``.
|
||||||
|
* The *target machine* can also be remote machine. (See the cli switch ``-r``)
|
||||||
|
* Remote machines are not limited to a specific architecture. All architectures supported by gluon can be used as remote machines.
|
||||||
|
|
||||||
|
* Finally opkg is used to install/update the packages in the target machine.
|
||||||
|
|
||||||
|
* While doing this, it will not override ``/etc/config`` with package defaults by default. (See the cli switch ``-P``).
|
||||||
|
* While doing this, opkg calls the ``check_site.lua`` from the package as post_install script to validate the ``site.conf``. This means that the ``site.conf`` of the target machine is used for this validation.
|
||||||
|
|
||||||
|
Note that:
|
||||||
|
|
||||||
|
* ``push_pkg.sh`` does neither build nor push dependencies of the packages automatically. If you want to update dependencies, you must explicitly specify them to be pushed.
|
||||||
|
* If you add new packages, you must run ``make update config GLUON_TARGET=...``.
|
||||||
|
* You can change the gluon target of the target machine via ``make config GLUON_TARGET=...``.
|
||||||
|
* If you want to update the ``site.conf`` of the target machine, use ``push_pkg.sh package/gluon-site/``.
|
||||||
|
* Sometimes when things break, you can heal them by compiling a package with its dependencies: ``cd openwrt; make package/gluon-ebtables/clean; make package/gluon-ebtables/compile; cd ..``.
|
||||||
|
* You can exit qemu by pressing ``CTRL + a`` and ``c`` afterwards.
|
||||||
|
|
||||||
Gluon package makefiles
|
Gluon package makefiles
|
||||||
=======================
|
=======================
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user