Merge pull request #527 from freifunk-gluon/modules

Generate opkg repo for kernel modules

.gitignore

@@ -1,6 +1,6 @@
 *~
 /build
-/images
+/output
 /site
 /openwrt
 /packages

Makefile

@@ -59,29 +59,37 @@ CheckTarget := [ -n '$(GLUON_TARGET)' -a -n '$(GLUON_TARGET_$(GLUON_TARGET)_BOAR
 CheckExternal := test -d $(GLUON_ORIGOPENWRTDIR) || (echo 'You don'"'"'t seem to have obtained the external repositories needed by Gluon; please call `make update` first!'; false)
 
 
+create-key: FORCE
+	@$(CheckExternal)
+	+@$(GLUONMAKE_EARLY) create-key
+
 prepare-target: FORCE
 	@$(CheckExternal)
 	@$(CheckTarget)
 	+@$(GLUONMAKE_EARLY) prepare-target
 
-
 all: prepare-target
+	+@$(GLUONMAKE) build-key
 	+@$(GLUONMAKE) prepare
 	+@$(GLUONMAKE) images
+	+@$(GLUONMAKE) modules
 
 prepare: prepare-target
+	+@$(GLUONMAKE) build-key
 	+@$(GLUONMAKE) $@
 
-clean download images: FORCE
+clean download images modules: FORCE
 	@$(CheckExternal)
 	@$(CheckTarget)
 	+@$(GLUONMAKE_EARLY) maybe-prepare-target
+	+@$(GLUONMAKE) build-key
 	+@$(GLUONMAKE) $@
 
 toolchain/% package/% target/% image/%: FORCE
 	@$(CheckExternal)
 	@$(CheckTarget)
 	+@$(GLUONMAKE_EARLY) maybe-prepare-target
+	+@$(GLUONMAKE) build-key
 	+@$(GLUONMAKE) $@
 
 manifest: FORCE
@@ -103,13 +111,6 @@ manifest: FORCE
 	mkdir -p $(GLUON_IMAGEDIR)/sysupgrade
 	mv $(GLUON_BUILDDIR)/$(GLUON_BRANCH).manifest.tmp $(GLUON_IMAGEDIR)/sysupgrade/$(GLUON_BRANCH).manifest
 
-update-vermagic: FORCE
-	@$(CheckExternal)
-	+($(foreach GLUON_TARGET,$(GLUON_TARGETS), \
-		$(GLUONMAKE_EARLY) maybe-prepare-target GLUON_TARGET='$(GLUON_TARGET)' V=s$(OPENWRT_VERBOSE) && \
-		$(GLUONMAKE) update-vermagic GLUON_TARGET='$(GLUON_TARGET)' V=s$(OPENWRT_VERBOSE) && \
-	) :)
-
 dirclean : FORCE
 	for dir in build_dir dl staging_dir tmp; do \
 		rm -rf $(GLUON_ORIGOPENWRTDIR)/$$dir; \
@@ -176,19 +177,6 @@ GLUON_$(1)_MODEL_$(2)_ALIASES += $(3)
 endef
 
 
-include $(GLUONDIR)/targets/targets.mk
-include $(GLUONDIR)/targets/$(GLUON_TARGET)/profiles.mk
-
-BOARD := $(GLUON_TARGET_$(GLUON_TARGET)_BOARD)
-override SUBTARGET := $(GLUON_TARGET_$(GLUON_TARGET)_SUBTARGET)
-
-target_prepared_stamp := $(BOARD_BUILDDIR)/target-prepared
-gluon_prepared_stamp := $(BOARD_BUILDDIR)/prepared
-
-
-include $(INCLUDE_DIR)/target.mk
-
-
 prereq: FORCE
 	+$(NO_TRACE_MAKE) prereq
 
@@ -216,7 +204,43 @@ feeds: FORCE
 
 gluon-tools: FORCE
 	+$(GLUONMAKE_EARLY) tools/sed/install
-	+$(GLUONMAKE_EARLY) package/lua/host/install
+	+$(GLUONMAKE_EARLY) package/lua/host/install package/usign/host/install
+
+
+prepare-early: FORCE
+	for dir in build_dir dl staging_dir; do \
+		mkdir -p $(GLUON_ORIGOPENWRTDIR)/$$dir; \
+	done
+
+	+$(GLUONMAKE_EARLY) feeds
+	+$(GLUONMAKE_EARLY) gluon-tools
+
+create-key: prepare-early
+	[ -s $(GLUON_OPKG_KEY) -a -s $(GLUON_OPKG_KEY).pub ] || \
+		$(STAGING_DIR_HOST)/bin/usign -G -s $(GLUON_OPKG_KEY) -p $(GLUON_OPKG_KEY).pub -c "Gluon opkg key"
+
+include $(GLUONDIR)/targets/targets.mk
+
+ifneq ($(GLUON_TARGET),)
+
+include $(GLUONDIR)/targets/$(GLUON_TARGET)/profiles.mk
+
+BOARD := $(GLUON_TARGET_$(GLUON_TARGET)_BOARD)
+override SUBTARGET := $(GLUON_TARGET_$(GLUON_TARGET)_SUBTARGET)
+
+target_prepared_stamp := $(BOARD_BUILDDIR)/target-prepared
+gluon_prepared_stamp := $(BOARD_BUILDDIR)/prepared
+
+PREPARED_RELEASE = $$(cat $(gluon_prepared_stamp))
+IMAGE_PREFIX = gluon-$(GLUON_SITE_CODE)-$(PREPARED_RELEASE)
+MODULE_PREFIX = gluon-$(GLUON_SITE_CODE)-$(PREPARED_RELEASE)
+
+
+include $(INCLUDE_DIR)/target.mk
+
+build-key: FORCE
+	ln -sf $(GLUON_OPKG_KEY) $(BUILD_KEY)
+	ln -sf $(GLUON_OPKG_KEY).pub $(BUILD_KEY).pub
 
 config: FORCE
 	+$(NO_TRACE_MAKE) scripts/config/conf OPENWRT_BUILD= QUIET=0
@@ -235,31 +259,19 @@ config: FORCE
 			| sed -e 's/ /\n/g'; \
 	) > $(BOARD_BUILDDIR)/config.tmp
 	scripts/config/conf --defconfig=$(BOARD_BUILDDIR)/config.tmp Config.in
-	mv .config $(BOARD_BUILDDIR)/config
 
-	echo 'CONFIG_ALL_KMODS=y' >> $(BOARD_BUILDDIR)/config.tmp
+prepare-target: create-key
-	scripts/config/conf --defconfig=$(BOARD_BUILDDIR)/config.tmp Config.in
-	mv .config $(BOARD_BUILDDIR)/config-allmods
-
-	cp $(BOARD_BUILDDIR)/config .config
-
-prepare-target: FORCE
 	rm $(GLUON_OPENWRTDIR)/tmp || true
 	mkdir -p $(GLUON_OPENWRTDIR)/tmp
 
-	for dir in build_dir dl staging_dir; do \
-		mkdir -p $(GLUON_ORIGOPENWRTDIR)/$$dir; \
-	done
 	for link in build_dir config Config.in dl include Makefile package rules.mk scripts staging_dir target toolchain tools; do \
 		ln -sf $(GLUON_ORIGOPENWRTDIR)/$$link $(GLUON_OPENWRTDIR); \
 	done
 
-	+$(GLUONMAKE_EARLY) feeds
-	+$(GLUONMAKE_EARLY) gluon-tools
 	+$(GLUONMAKE) config
 	touch $(target_prepared_stamp)
 
-$(target_prepared_stamp):
+$(target_prepared_stamp): create-key
 	+$(GLUONMAKE_EARLY) prepare-target
 
 maybe-prepare-target: $(target_prepared_stamp)
@@ -291,8 +303,8 @@ toolchain: $(toolchain/stamp-install) $(tools/stamp-install)
 include $(INCLUDE_DIR)/kernel.mk
 
 kernel: FORCE
-	+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) -f $(GLUONDIR)/include/Makefile.target $(LINUX_DIR)/.image TARGET_BUILD=1
+	+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) $(LINUX_DIR)/.image TARGET_BUILD=1
-	+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) -f $(GLUONDIR)/include/Makefile.target $(LINUX_DIR)/.modules TARGET_BUILD=1
+	+$(NO_TRACE_MAKE) -C $(TOPDIR)/target/linux/$(BOARD) $(LINUX_DIR)/.modules TARGET_BUILD=1
 
 packages: $(package/stamp-compile)
 	$(_SINGLE)$(SUBMAKE) -r package/index
@@ -320,6 +332,14 @@ prepare: FORCE
 $(gluon_prepared_stamp):
 	+$(GLUONMAKE) prepare
 
+modules: FORCE $(gluon_prepared_stamp)
+	-rm -f $(GLUON_MODULEDIR)/*/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)/*
+	-rmdir -p $(GLUON_MODULEDIR)/*/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
+	mkdir -p $(GLUON_MODULEDIR)/$(MODULE_PREFIX)/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
+	cp $(PACKAGE_DIR)/kmod-*.ipk $(GLUON_MODULEDIR)/$(MODULE_PREFIX)/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
+
+	$(_SINGLE)$(SUBMAKE) -r package/index PACKAGE_DIR=$(GLUON_MODULEDIR)/$(MODULE_PREFIX)/$(BOARD)/$(if $(SUBTARGET),$(SUBTARGET),generic)
+
 
 include $(INCLUDE_DIR)/package-ipkg.mk
 
@@ -332,9 +352,6 @@ BIN_DIR = $(PROFILE_BUILDDIR)/images
 
 TARGET_DIR = $(PROFILE_BUILDDIR)/root
 
-PREPARED_RELEASE = $$(cat $(gluon_prepared_stamp))
-IMAGE_PREFIX = gluon-$(GLUON_SITE_CODE)-$(PREPARED_RELEASE)
-
 OPKG:= \
   IPKG_TMP="$(TMP_DIR)/ipkgtmp" \
   IPKG_INSTROOT="$(TARGET_DIR)" \
@@ -447,15 +464,7 @@ manifest: FORCE
 		) : \
 	) >> $(GLUON_BUILDDIR)/$(GLUON_BRANCH).manifest.tmp
 
-update-vermagic: FORCE
+.PHONY: all create-key prepare images modules clean gluon-tools manifest
-	mkdir -p '$(BOARD_BUILDDIR)'
-	echo '$(DEFAULT_OPKG_REPO)' > '$(BOARD_BUILDDIR)/default_opkg_repo'
-	$(VERSION_SED) '$(BOARD_BUILDDIR)/default_opkg_repo'
-	wget -q -O- "$$(cat '$(BOARD_BUILDDIR)/default_opkg_repo')/base/Packages.gz" \
-		| gzip -d \
-		| awk '/Depends: kernel / { match($$3,/[[:xdigit:]]{32}/,m); print m[0]; exit }' \
-		> $(GLUONDIR)/targets/$(GLUON_TARGET)/vermagic
-
-.PHONY: all images prepare clean gluon-tools manifest update-vermagic
 
 endif
+endif

docs/dev/hardware.rst

@@ -76,21 +76,9 @@ A directory for the new target must be created under ``targets``, and it must be
 to ``targets/targets.mk``. In the new target directory, the following files must be created:
 
 * profiles.mk
-* vermagic
 * config (optional)
 
 For ``profiles.mk``, see :ref:`hardware-adding-profiles`.
 The file ``config`` can be used to add additional, target-specific options to the OpenWrt config.
 
-The files ``vermagic`` must have the correct content so kernel modules from the upstream repositories
-can be installed without dependency issues. The OpenWrt version a Gluon release is based on is defined by the upstream package repo URL in ``include/gluon.mk``
-(in the variable ``DEFAULT_OPKG_REPO``); at the time this documentation was written, this was ``barrier_breaker/14.07``; whenever
-the package repo is updated, all ``vermagic`` files must be updated as well. The command ``make update-vermagic`` can be used to get the correct
-vermagic, or update the values when ``DEFAULT_OPKG_REPO`` has changed.
-
-The content is a hash which is part of the version number of the kernel package. So in the case of ``ar71xx-generic`` on
-``barrier_breaker``, we look for the kernel package in the directory ``https://downloads.openwrt.org/barrier_breaker/14.07/ar71xx/generic/packages/base/``.
-As the kernel package is called ``kernel_3.10.49-1-0114c71ed85677c9c1e4911437af4743_ar71xx.ipk``, the correct ``vermagic`` string
-is ``0114c71ed85677c9c1e4911437af4743``.
-
 After this, is should be sufficient to call ``make GLUON_TARGET=<target>`` to build the images for the new target.

docs/features/autoupdater.rst

@@ -30,10 +30,10 @@ A fully automated nightly build could use the following commands:
     make clean
     make -j5 GLUON_TARGET=ar71xx-generic GLUON_BRANCH=experimental
     make manifest GLUON_BRANCH=experimental
-    contrib/sign.sh $SECRETKEY images/sysupgrade/experimental.manifest
+    contrib/sign.sh $SECRETKEY output/images/sysupgrade/experimental.manifest
 
     rm -rf /where/to/put/this/experimental
-    cp -r images /where/to/put/this/experimental
+    cp -r output/images /where/to/put/this/experimental
 
 
 Infrastructure

docs/user/getting_started.rst

@@ -86,7 +86,7 @@ In case of errors read the messages carefully and try to fix the stated issues (
 ``ar71xx-generic`` is the most common target and will generated images for most of the supported hardware.
 To see a complete list of supported targets, call ``make`` without setting ``GLUON_TARGET``.
 
-The built images can be found in the directory `images`. Of these, the factory
+The built images can be found in the directory `output/images`. Of these, the factory
 images are to be used when flashing from the original firmware a device came with,
 and sysupgrade is to upgrade from other versions of Gluon or any other OpenWRT-based
 system.
@@ -109,19 +109,63 @@ will clean the entire tree, so the toolchain will be rebuilt as well, which is
 not necessary in most cases, and will take a while.
 
 
+opkg repositories
+-----------------
+
+Gluon is mostly compatible with OpenWrt, so the normal OpenWrt package repositories
+can be used for Gluon as well. It is advisable to setup a mirror or reverse proxy
+reachable over IPv6 and add it to ``site.conf`` as http://downloads.openwrt.org/ does
+not support IPv6.
+
+This is not true for kernel modules; the Gluon kernel is incompatible with the
+kernel of the default OpenWrt images. Therefore, Gluon will not only generate images,
+but also an opkg repositoy containing all kernel modules provided by OpenWrt/Gluon
+for the kernel of the generated images.
+
+Signing keys
+............
+
+Gluon does not support HTTPS for downloading packages; fortunately, opkg deploys
+public-key cryptography to ensure package integrity.
+
+The Gluon images will contain two public keys: the official OpenWrt signing key
+(to allow installing userspace packages) and a Gluon-specific key (which is used
+to sign the generated module repository).
+
+By default, Gluon will handle the generation and handling of the keys itself.
+When making firmware releases based on Gluon, it might make sense to store the
+the keypair, so updating the module repository later is possible.
+
+The location the keys are stored at and read from can be changed
+(see :ref:`getting-started-environment-variables`). To only generate the keypair
+at the configured location without doing a full build, use ``make create-key``.
+
+.. _getting-started-environment-variables:
+
 Environment variables
 ---------------------
 
 Gluon's build process can be controlled by various environment variables.
 
 GLUON_SITEDIR
-  Path to the site configuration. Defaults to ``site/``.
+  Path to the site configuration. Defaults to ``site``.
-
-GLUON_IMAGEDIR
-  Path where images will be stored. Defaults to ``images/``.
 
 GLUON_BUILDDIR
-  Working directory during build. Defaults to ``build/``.
+  Working directory during build. Defaults to ``build``.
+
+GLUON_OPKG_KEY
+  Path key file used to sign the module opkg repository. Defaults to ``$(GLUON_BULDDIR)/gluon-opkg-key``.
+
+  The private key will be stored as ``$(GLUON_OPKG_KEY)``, the public key as ``$(GLUON_OPKG_KEY).pub``.
+
+GLUON_OUTPUTDIR
+  Path where output files will be stored. Defaults to ``output``.
+
+GLUON_IMAGEDIR
+  Path where images will be stored. Defaults to ``$(GLUON_OUTPUTDIR)/images``.
+
+GLUON_MODULEDIR
+  Path where the kernel module opkg repository will be stored. Defaults to ``$(GLUON_OUTPUTDIR)/modules``.
 
 
 So all in all, to update and rebuild a Gluon build tree, the following commands should be used (repeat the

include/Makefile.target

@@ -1,19 +0,0 @@
-# code adjusted from openwrt/include/kernel-defaults.mk
-
-override define Kernel/Configure
-	$(LINUX_CONF_CMD) > $(LINUX_DIR)/.config.target
-# copy CONFIG_KERNEL_* settings over to .config.target
-	awk '/^(#[[:space:]]+)?CONFIG_KERNEL/{sub("CONFIG_KERNEL_","CONFIG_");print}' $(BOARD_BUILDDIR)/config-allmods >> $(LINUX_DIR)/.config.target
-	echo "# CONFIG_KALLSYMS_EXTRA_PASS is not set" >> $(LINUX_DIR)/.config.target
-	echo "# CONFIG_KALLSYMS_ALL is not set" >> $(LINUX_DIR)/.config.target
-	echo "# CONFIG_KALLSYMS_UNCOMPRESSED is not set" >> $(LINUX_DIR)/.config.target
-	$(SCRIPT_DIR)/metadata.pl kconfig $(TMP_DIR)/.packageinfo $(BOARD_BUILDDIR)/config-allmods $(KERNEL_PATCHVER) > $(LINUX_DIR)/.config.override
-	$(SCRIPT_DIR)/kconfig.pl 'm+' '+' $(LINUX_DIR)/.config.target /dev/null $(LINUX_DIR)/.config.override > $(LINUX_DIR)/.config
-	$(call Kernel/SetNoInitramfs)
-	rm -rf $(KERNEL_BUILD_DIR)/modules
-	$(_SINGLE) [ -d $(LINUX_DIR)/user_headers ] || $(MAKE) $(KERNEL_MAKEOPTS) INSTALL_HDR_PATH=$(LINUX_DIR)/user_headers headers_install
-	cp $(GLUONDIR)/targets/$(GLUON_TARGET)/vermagic $(LINUX_DIR)/.vermagic
-endef
-
-# The Makefile included here is $(TOPDIR)/target/linux/$(BOARD)/Makefile
-include Makefile

include/config

@@ -1,7 +1,7 @@
 CONFIG_IMAGEOPT=y
 # CONFIG_PER_FEED_REPO is not set
-
 CONFIG_DEVEL=y
+CONFIG_ALL_KMODS=y
 
 CONFIG_BUSYBOX_CUSTOM=y
 CONFIG_BUSYBOX_CONFIG_SHA512SUM=y

include/gluon.mk

@@ -2,13 +2,18 @@ ifneq ($(__gluon_inc),1)
 __gluon_inc=1
 
 GLUON_SITEDIR ?= $(GLUONDIR)/site
-GLUON_IMAGEDIR ?= $(GLUONDIR)/images
 GLUON_BUILDDIR ?= $(GLUONDIR)/build
 
 GLUON_ORIGOPENWRTDIR := $(GLUONDIR)/openwrt
 GLUON_SITE_CONFIG := $(GLUON_SITEDIR)/site.conf
 
-export GLUONDIR GLUON_SITEDIR GLUON_SITE_CONFIG GLUON_IMAGEDIR GLUON_BUILDDIR
+GLUON_OUTPUTDIR ?= $(GLUONDIR)/output
+GLUON_IMAGEDIR ?= $(GLUON_OUTPUTDIR)/images
+GLUON_MODULEDIR ?= $(GLUON_OUTPUTDIR)/modules
+
+GLUON_OPKG_KEY ?= $(GLUON_BUILDDIR)/gluon-opkg-key
+
+export GLUONDIR GLUON_SITEDIR GLUON_BUILDDIR GLUON_SITE_CONFIG GLUON_OUTPUTDIR GLUON_IMAGEDIR GLUON_MODULEDIR
 
 
 BOARD_BUILDDIR = $(GLUON_BUILDDIR)/$(GLUON_TARGET)
@@ -16,6 +21,11 @@ BOARD_KDIR = $(BOARD_BUILDDIR)/kernel
 
 export BOARD_BUILDDIR
 
+
+LINUX_RELEASE := 2
+export LINUX_RELEASE
+
+
 GLUON_OPENWRTDIR = $(BOARD_BUILDDIR)/openwrt
 
 

targets/ar71xx-generic/vermagic

@@ -1 +0,0 @@
-7bed08fa9c06eb8089e82c200340ec66

targets/ar71xx-nand/vermagic

@@ -1 +0,0 @@
-a44d8090b8f752ec1d96871dbbb8988a

targets/brcm2708-bcm2708/vermagic

@@ -1 +0,0 @@
-57d4b7d13c23edd1a689dd6d6adc65ad

targets/brcm2708-bcm2709/vermagic

@@ -1 +0,0 @@
-0fec8065ae15bf9934ffbe2512e8e39f

targets/mpc85xx-generic/vermagic

@@ -1 +0,0 @@
-521812ea85032e662b3d3469a3a10df8

targets/ramips-rt305x/vermagic

@@ -1 +0,0 @@
-c8b57a131072a3198e594822481af3e0

targets/sunxi/vermagic

@@ -1 +0,0 @@
-3c199b535784c56bd7a6b4d6ad82b91f

targets/x86-64/vermagic

@@ -1 +0,0 @@
-30da46d39f906146155850351fa0acd9

targets/x86-generic/vermagic

@@ -1 +0,0 @@
-8549f8163c15d79b053f26aa0d52e96f

targets/x86-kvm_guest/vermagic

@@ -1 +0,0 @@
-d0d2aab265f2535385cf8d756315d27c

targets/x86-xen_domu/vermagic

@@ -1 +0,0 @@
-bce85557e89d5a4f30d6d723503b1a61