build: move package list generation to target_config_lib.lua, fix precedence

The precedence of different package lists was broken since #1876,
disallowing removal of GLUON_FEATURES packages via GLUON_SITE_PACKAGES.

Including all package selections, both implicit defaults and explicit
handling in Gluon, the order of precedence is now the following:

1.  OpenWrt defaults (including target-specific defaults)
2.  Device-specific packages from OpenWrt
3.  Generic default packages (from target/generic)
4.  Target default packages (target/$(GLUON_TARGET))
5.  Removal of opkg for tiny targets
6.  Packages derived from GLUON_FEATURES + GLUON_FEATURES_$(class)
7.  GLUON_SITE_PACKAGES
8.  GLUON_SITE_PACKAGES_$(class)
9.  Device-specific packages from target/$(GLUON_TARGET)
10. Device-specific packages from GLUON_$(device)_SITE_PACKAGES

This also contains various pieces of cleanup:

- No hardcoded order of device classes for target_config.lua arguments
  anymore (in fact, the Makefile doesn't know anything about device
  classes now)
- target_conifg_lib.lua only hardcodes the fallback class for x86, no
  other occurences of specific class names
- Feature -> package list mapping is moved from Makefile to the Lua code
  as well (still implemented in Shell though)

Makefile

@@ -55,7 +55,8 @@ endef
 GLUON_VARS = \
 	GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_DEBUG GLUON_DEPRECATED GLUON_DEVICES \
 	GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR \
-	GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS BOARD SUBTARGET
+	GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS \
+	GLUON_TARGET BOARD SUBTARGET
 
 unexport $(GLUON_VARS)
 GLUON_ENV = $(foreach var,$(GLUON_VARS),$(var)=$(call escape,$($(var))))
@@ -135,31 +136,6 @@ lint-lua: FORCE
 lint-sh: FORCE
 	scripts/lint-sh.sh
 
-define merge_lists
-  $(1) :=
-  $(foreach pkg,$(2),
-    $(1) := $$(strip $$(filter-out -$$(patsubst -%,%,$(pkg)) $$(patsubst -%,%,$(pkg)),$$(value $(1))) $(pkg))
-  )
-endef
-
-define feature_packages
-  $(1) := $(shell scripts/features.sh '$(2)' || echo '__ERROR__')
-endef
-
-$(eval $(call merge_lists,GLUON_FEATURE_LIST_standard,$(GLUON_FEATURES) $(GLUON_FEATURES_standard)))
-$(eval $(call merge_lists,GLUON_FEATURE_LIST_tiny,$(GLUON_FEATURES) $(GLUON_FEATURES_tiny)))
-
-$(eval $(call feature_packages,GLUON_FEATURE_PACKAGES_standard,$(GLUON_FEATURE_LIST_standard)))
-$(eval $(call feature_packages,GLUON_FEATURE_PACKAGES_tiny,$(GLUON_FEATURE_LIST_tiny)))
-
-ifneq ($(filter __ERROR__,$(GLUON_FEATURES_standard) $(GLUON_FEATURES_tiny)),)
-  $(error Error while evaluating features)
-endif
-
-$(eval $(call merge_lists,GLUON_DEFAULT_PACKAGES,$(GLUON_DEFAULT_PACKAGES) $(GLUON_SITE_PACKAGES)))
-$(eval $(call merge_lists,GLUON_CLASS_PACKAGES_standard,$(GLUON_FEATURE_PACKAGES_standard) $(GLUON_SITE_PACKAGES_standard)))
-$(eval $(call merge_lists,GLUON_CLASS_PACKAGES_tiny,$(GLUON_FEATURE_PACKAGES_tiny) $(GLUON_SITE_PACKAGES_tiny)))
-
 
 LUA := openwrt/staging_dir/hostpkg/bin/lua
 
@@ -182,22 +158,16 @@ config: $(LUA) FORCE
 		$(call CheckSite,$(conf)); \
 	)
 
-	$(GLUON_ENV) \
+	$(GLUON_ENV) $(LUA) scripts/target_config.lua > openwrt/.config
-		$(LUA) scripts/target_config.lua '$(GLUON_TARGET)' '$(GLUON_DEFAULT_PACKAGES)' '$(GLUON_CLASS_PACKAGES_standard)' '$(GLUON_CLASS_PACKAGES_tiny)' \
-		> openwrt/.config
 	$(OPENWRTMAKE) defconfig
-
+	$(GLUON_ENV) $(LUA) scripts/target_config_check.lua
-	$(GLUON_ENV) \
-		$(LUA) scripts/target_config_check.lua '$(GLUON_TARGET)' '$(GLUON_DEFAULT_PACKAGES)' '$(GLUON_CLASS_PACKAGES_standard)' '$(GLUON_CLASS_PACKAGES_tiny)'
 
 
 all: config
 	+
-	$(GLUON_ENV) \
+	$(GLUON_ENV) $(LUA) scripts/clean_output.lua
-		$(LUA) scripts/clean_output.lua
 	$(OPENWRTMAKE)
-	$(GLUON_ENV) \
+	$(GLUON_ENV) $(LUA) scripts/copy_output.lua
-		$(LUA) scripts/copy_output.lua '$(GLUON_TARGET)'
 
 clean download: config
 	+$(OPENWRTMAKE) $@

scripts/copy_output.lua

@@ -1,12 +1,13 @@
 local lib = dofile('scripts/target_lib.lua')
 local env = lib.env
 
+local target = env.GLUON_TARGET
+
+assert(target)
 assert(env.GLUON_IMAGEDIR)
 assert(env.GLUON_PACKAGEDIR)
 
 
-local target = arg[1]
-
 local openwrt_target
 local subtarget = env.SUBTARGET
 if subtarget ~= '' then

scripts/target_config_lib.lua

@@ -1,17 +1,55 @@
+-- Split a string into words
+local function split(s)
+	local ret = {}
+	for w in string.gmatch(s, '%S+') do
+		table.insert(ret, w)
+	end
+	return ret
+end
+
+-- Strip leading '-' character
+local function strip_neg(s)
+	if string.sub(s, 1, 1) == '-' then
+		return string.sub(s, 2)
+	else
+		return s
+	end
+end
+
+-- Add an element to a list, removing duplicate entries and handling negative
+-- elements prefixed with a '-'
+local function append_to_list(list, item, keep_neg)
+	local match = strip_neg(item)
+	local ret = {}
+	for _, el in ipairs(list) do
+		if strip_neg(el) ~= match then
+			table.insert(ret, el)
+		end
+	end
+	if keep_neg ~= false or string.sub(item, 1, 1) ~= '-' then
+		table.insert(ret, item)
+	end
+	return ret
+end
+
+local function compact_list(list, keep_neg)
+	local ret = {}
+	for _, el in ipairs(list) do
+		ret  = append_to_list(ret, el, keep_neg)
+	end
+	return ret
+end
+
 return function(funcs)
 	local lib = dofile('scripts/target_lib.lua')
 	local env = lib.env
 
+	local target = env.GLUON_TARGET
+
+	assert(target)
 	assert(env.BOARD)
 	assert(env.SUBTARGET)
 
-	local target = arg[1]
-	local extra_packages = arg[2]
-	local class_packages = {
-		standard = arg[3],
-		tiny = arg[4],
-	}
-
 	local openwrt_config_target
 	if env.SUBTARGET ~= '' then
 		openwrt_config_target = env.BOARD .. '_' .. env.SUBTARGET
@@ -32,73 +70,102 @@ END_MAKE
 	end
 
 	local function site_packages(image)
-		return site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image))
+		return split(site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image)))
+	end
+
+	-- TODO: Rewrite features.sh in Lua
+	local function feature_packages(features)
+		-- Ugly hack: Lua doesn't give us the return code of a popened
+		-- command, so we match on a special __ERROR__ marker
+		local pkgs = lib.exec_capture({'scripts/features.sh', features}, '|| echo __ERROR__')
+		assert(string.find(pkgs, '__ERROR__') == nil, 'Error while evaluating features')
+		return pkgs
+	end
+
+	-- This involves running lots of processes to evaluate site.mk, so we
+	-- add a simple cache
+	local class_cache = {}
+	local function class_packages(class)
+		if class_cache[class] then
+			return class_cache[class]
+		end
+
+		local features = site_vars(string.format('$(GLUON_FEATURES) $(GLUON_FEATURES_%s)', class))
+		features = table.concat(compact_list(split(features), false), ' ')
+
+		local pkgs = feature_packages(features)
+		pkgs = pkgs .. ' ' .. site_vars(string.format('$(GLUON_SITE_PACKAGES) $(GLUON_SITE_PACKAGES_%s)', class))
+
+		pkgs = compact_list(split(pkgs))
+
+		class_cache[class] = pkgs
+		return pkgs
 	end
 
 	local function handle_target_pkgs(pkgs)
-		local packages = string.gmatch(pkgs, '%S+')
+		for _, pkg in ipairs(pkgs) do
-		for pkg in packages do
+			if string.sub(pkg, 1, 1) == '-' then
-			lib.packages {pkg}
+				lib.try_config('# CONFIG_PACKAGE_%s is not set', string.sub(pkg, 2))
+			else
+				funcs.config_package(lib.config, pkg, 'y')
+			end
 		end
 	end
 
 	lib.include('generic')
-	handle_target_pkgs(extra_packages)
 	lib.include(target)
 
-	if lib.target_class ~= nil then
-		handle_target_pkgs(class_packages[lib.target_class])
-	end
-
 	lib.check_devices()
 
-
 	if not lib.opkg then
 		lib.config '# CONFIG_SIGNED_PACKAGES is not set'
 		lib.config 'CONFIG_CLEAN_IPKG=y'
 		lib.packages {'-opkg'}
 	end
 
+	if #lib.devices > 0 then
+		handle_target_pkgs(lib.target_packages)
 
-	local default_pkgs = ''
+		for _, dev in ipairs(lib.devices) do
-	for _, pkg in ipairs(lib.target_packages) do
+			local profile = dev.options.profile or dev.name
-		default_pkgs = default_pkgs .. ' ' .. pkg
 
-		if string.sub(pkg, 1, 1) == '-' then
+			local device_pkgs = {}
-			lib.try_config('# CONFIG_PACKAGE_%s is not set', string.sub(pkg, 2))
+			local function handle_pkgs(pkgs)
-		else
+				for _, pkg in ipairs(pkgs) do
-			funcs.config_package(lib.config, pkg, 'y')
+					if string.sub(pkg, 1, 1) ~= '-' then
-		end
+						funcs.config_package(lib.config, pkg, 'm')
-	end
+					end
-
+					device_pkgs = append_to_list(device_pkgs, pkg)
-	for _, dev in ipairs(lib.devices) do
+				end
-		local profile = dev.options.profile or dev.name
-		local device_pkgs = default_pkgs
-
-		local function handle_pkg(pkg)
-			if string.sub(pkg, 1, 1) ~= '-' then
-				funcs.config_package(lib.config, pkg, 'm')
 			end
-			device_pkgs = device_pkgs .. ' ' .. pkg
+
+			handle_pkgs(lib.target_packages)
+			handle_pkgs(class_packages(dev.options.class))
+			handle_pkgs(dev.options.packages or {})
+			handle_pkgs(site_packages(dev.image))
+
+			funcs.config_message(lib.config, string.format("unable to enable device '%s'", profile),
+				'CONFIG_TARGET_DEVICE_%s_DEVICE_%s=y', openwrt_config_target, profile)
+			lib.config('CONFIG_TARGET_DEVICE_PACKAGES_%s_DEVICE_%s="%s"',
+				openwrt_config_target, profile,
+				table.concat(device_pkgs, ' '))
 		end
+	else
+		-- x86 fallback: no devices
+		local target_pkgs = {}
 		local function handle_pkgs(pkgs)
-			local packages = string.gmatch(pkgs or '', '%S+')
+			for _, pkg in ipairs(pkgs) do
-			for pkg in packages do
+				target_pkgs = append_to_list(target_pkgs, pkg)
-				handle_pkg(pkg)
 			end
 		end
 
-		for _, pkg in ipairs(dev.options.packages or {}) do
+		-- Just hardcode the class for device-less targets to 'standard'
-			handle_pkg(pkg)
+		-- - this is x86 only at the moment, and it will have devices
-		end
+		-- in OpenWrt 19.07 + 1 as well
-		handle_pkgs(site_packages(dev.image))
+		handle_pkgs(lib.target_packages)
+		handle_pkgs(class_packages('standard'))
 
-		handle_pkgs(class_packages[dev.options.class])
+		handle_target_pkgs(target_pkgs)
-
-		funcs.config_message(lib.config, string.format("unable to enable device '%s'", profile),
-			'CONFIG_TARGET_DEVICE_%s_DEVICE_%s=y', openwrt_config_target, profile)
-		lib.config('CONFIG_TARGET_DEVICE_PACKAGES_%s_DEVICE_%s="%s"',
-			openwrt_config_target, profile, device_pkgs)
 	end
 
 	return lib

scripts/target_lib.lua

@@ -23,7 +23,6 @@ assert(env.GLUON_DEPRECATED)
 
 M.site_code = assert(assert(dofile('scripts/site_config.lua')('site.conf')).site_code)
 M.target_packages = {}
-M.target_class = nil
 M.configs = {}
 M.devices = {}
 M.images = {}
@@ -40,7 +39,7 @@ local default_options = {
 	aliases = {},
 	manifest_aliases = {},
 	packages = {},
-	class = "standard",
+	class = 'standard',
 	deprecated = false,
 	broken = false,
 }
@@ -159,10 +158,6 @@ function F.config(...)
 	M.configs[string.format(...)] = 2
 end
 
-function F.class(target_class)
-	M.target_class = target_class
-end
-
 function F.packages(pkgs)
 	for _, pkg in ipairs(pkgs) do
 		table.insert(M.target_packages, pkg)

targets/x86-geode

@@ -1,5 +1,3 @@
-class 'standard'
-
 packages {
 	'kmod-3c59x',
 	'kmod-8139cp',

targets/x86.inc

@@ -1,8 +1,6 @@
 config 'CONFIG_VDI_IMAGES=y'
 config 'CONFIG_VMDK_IMAGES=y'
 
-class 'standard'
-
 packages {
 	'kmod-3c59x',
 	'kmod-8139cp',