From 53690d83be1ddb56c145c4e678e44632167fba71 Mon Sep 17 00:00:00 2001 From: Matthias Schiffer Date: Sun, 3 May 2020 21:35:01 +0200 Subject: [PATCH] build: move package list generation to target_config_lib.lua, fix precedence The precedence of different package lists was broken since #1876, disallowing removal of GLUON_FEATURES packages via GLUON_SITE_PACKAGES. Including all package selections, both implicit defaults and explicit handling in Gluon, the order of precedence is now the following: 1. OpenWrt defaults (including target-specific defaults) 2. Device-specific packages from OpenWrt 3. Generic default packages (from target/generic) 4. Target default packages (target/$(GLUON_TARGET)) 5. Removal of opkg for tiny targets 6. Packages derived from GLUON_FEATURES + GLUON_FEATURES_$(class) 7. GLUON_SITE_PACKAGES 8. GLUON_SITE_PACKAGES_$(class) 9. Device-specific packages from target/$(GLUON_TARGET) 10. Device-specific packages from GLUON_$(device)_SITE_PACKAGES This also contains various pieces of cleanup: - No hardcoded order of device classes for target_config.lua arguments anymore (in fact, the Makefile doesn't know anything about device classes now) - target_conifg_lib.lua only hardcodes the fallback class for x86, no other occurences of specific class names - Feature -> package list mapping is moved from Makefile to the Lua code as well (still implemented in Shell though) --- Makefile | 42 ++------- scripts/copy_output.lua | 5 +- scripts/target_config_lib.lua | 163 ++++++++++++++++++++++++---------- scripts/target_lib.lua | 7 +- targets/x86-geode | 2 - targets/x86.inc | 2 - 6 files changed, 125 insertions(+), 96 deletions(-) diff --git a/Makefile b/Makefile index ab52cc9a..f614f3d9 100644 --- a/Makefile +++ b/Makefile @@ -55,7 +55,8 @@ endef GLUON_VARS = \ GLUON_RELEASE GLUON_REGION GLUON_MULTIDOMAIN GLUON_DEBUG GLUON_DEPRECATED GLUON_DEVICES \ GLUON_TARGETSDIR GLUON_PATCHESDIR GLUON_TMPDIR GLUON_IMAGEDIR GLUON_PACKAGEDIR \ - GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS BOARD SUBTARGET + GLUON_SITEDIR GLUON_RELEASE GLUON_BRANCH GLUON_LANGS GLUON_BASE_FEEDS \ + GLUON_TARGET BOARD SUBTARGET unexport $(GLUON_VARS) GLUON_ENV = $(foreach var,$(GLUON_VARS),$(var)=$(call escape,$($(var)))) @@ -135,31 +136,6 @@ lint-lua: FORCE lint-sh: FORCE scripts/lint-sh.sh -define merge_lists - $(1) := - $(foreach pkg,$(2), - $(1) := $$(strip $$(filter-out -$$(patsubst -%,%,$(pkg)) $$(patsubst -%,%,$(pkg)),$$(value $(1))) $(pkg)) - ) -endef - -define feature_packages - $(1) := $(shell scripts/features.sh '$(2)' || echo '__ERROR__') -endef - -$(eval $(call merge_lists,GLUON_FEATURE_LIST_standard,$(GLUON_FEATURES) $(GLUON_FEATURES_standard))) -$(eval $(call merge_lists,GLUON_FEATURE_LIST_tiny,$(GLUON_FEATURES) $(GLUON_FEATURES_tiny))) - -$(eval $(call feature_packages,GLUON_FEATURE_PACKAGES_standard,$(GLUON_FEATURE_LIST_standard))) -$(eval $(call feature_packages,GLUON_FEATURE_PACKAGES_tiny,$(GLUON_FEATURE_LIST_tiny))) - -ifneq ($(filter __ERROR__,$(GLUON_FEATURES_standard) $(GLUON_FEATURES_tiny)),) - $(error Error while evaluating features) -endif - -$(eval $(call merge_lists,GLUON_DEFAULT_PACKAGES,$(GLUON_DEFAULT_PACKAGES) $(GLUON_SITE_PACKAGES))) -$(eval $(call merge_lists,GLUON_CLASS_PACKAGES_standard,$(GLUON_FEATURE_PACKAGES_standard) $(GLUON_SITE_PACKAGES_standard))) -$(eval $(call merge_lists,GLUON_CLASS_PACKAGES_tiny,$(GLUON_FEATURE_PACKAGES_tiny) $(GLUON_SITE_PACKAGES_tiny))) - LUA := openwrt/staging_dir/hostpkg/bin/lua @@ -182,22 +158,16 @@ config: $(LUA) FORCE $(call CheckSite,$(conf)); \ ) - $(GLUON_ENV) \ - $(LUA) scripts/target_config.lua '$(GLUON_TARGET)' '$(GLUON_DEFAULT_PACKAGES)' '$(GLUON_CLASS_PACKAGES_standard)' '$(GLUON_CLASS_PACKAGES_tiny)' \ - > openwrt/.config + $(GLUON_ENV) $(LUA) scripts/target_config.lua > openwrt/.config $(OPENWRTMAKE) defconfig - - $(GLUON_ENV) \ - $(LUA) scripts/target_config_check.lua '$(GLUON_TARGET)' '$(GLUON_DEFAULT_PACKAGES)' '$(GLUON_CLASS_PACKAGES_standard)' '$(GLUON_CLASS_PACKAGES_tiny)' + $(GLUON_ENV) $(LUA) scripts/target_config_check.lua all: config + - $(GLUON_ENV) \ - $(LUA) scripts/clean_output.lua + $(GLUON_ENV) $(LUA) scripts/clean_output.lua $(OPENWRTMAKE) - $(GLUON_ENV) \ - $(LUA) scripts/copy_output.lua '$(GLUON_TARGET)' + $(GLUON_ENV) $(LUA) scripts/copy_output.lua clean download: config +$(OPENWRTMAKE) $@ diff --git a/scripts/copy_output.lua b/scripts/copy_output.lua index 78300aa7..91547964 100755 --- a/scripts/copy_output.lua +++ b/scripts/copy_output.lua @@ -1,12 +1,13 @@ local lib = dofile('scripts/target_lib.lua') local env = lib.env +local target = env.GLUON_TARGET + +assert(target) assert(env.GLUON_IMAGEDIR) assert(env.GLUON_PACKAGEDIR) -local target = arg[1] - local openwrt_target local subtarget = env.SUBTARGET if subtarget ~= '' then diff --git a/scripts/target_config_lib.lua b/scripts/target_config_lib.lua index c52b7e30..ee2bf006 100644 --- a/scripts/target_config_lib.lua +++ b/scripts/target_config_lib.lua @@ -1,17 +1,55 @@ +-- Split a string into words +local function split(s) + local ret = {} + for w in string.gmatch(s, '%S+') do + table.insert(ret, w) + end + return ret +end + +-- Strip leading '-' character +local function strip_neg(s) + if string.sub(s, 1, 1) == '-' then + return string.sub(s, 2) + else + return s + end +end + +-- Add an element to a list, removing duplicate entries and handling negative +-- elements prefixed with a '-' +local function append_to_list(list, item, keep_neg) + local match = strip_neg(item) + local ret = {} + for _, el in ipairs(list) do + if strip_neg(el) ~= match then + table.insert(ret, el) + end + end + if keep_neg ~= false or string.sub(item, 1, 1) ~= '-' then + table.insert(ret, item) + end + return ret +end + +local function compact_list(list, keep_neg) + local ret = {} + for _, el in ipairs(list) do + ret = append_to_list(ret, el, keep_neg) + end + return ret +end + return function(funcs) local lib = dofile('scripts/target_lib.lua') local env = lib.env + local target = env.GLUON_TARGET + + assert(target) assert(env.BOARD) assert(env.SUBTARGET) - local target = arg[1] - local extra_packages = arg[2] - local class_packages = { - standard = arg[3], - tiny = arg[4], - } - local openwrt_config_target if env.SUBTARGET ~= '' then openwrt_config_target = env.BOARD .. '_' .. env.SUBTARGET @@ -32,73 +70,102 @@ END_MAKE end local function site_packages(image) - return site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image)) + return split(site_vars(string.format('$(GLUON_%s_SITE_PACKAGES)', image))) + end + + -- TODO: Rewrite features.sh in Lua + local function feature_packages(features) + -- Ugly hack: Lua doesn't give us the return code of a popened + -- command, so we match on a special __ERROR__ marker + local pkgs = lib.exec_capture({'scripts/features.sh', features}, '|| echo __ERROR__') + assert(string.find(pkgs, '__ERROR__') == nil, 'Error while evaluating features') + return pkgs + end + + -- This involves running lots of processes to evaluate site.mk, so we + -- add a simple cache + local class_cache = {} + local function class_packages(class) + if class_cache[class] then + return class_cache[class] + end + + local features = site_vars(string.format('$(GLUON_FEATURES) $(GLUON_FEATURES_%s)', class)) + features = table.concat(compact_list(split(features), false), ' ') + + local pkgs = feature_packages(features) + pkgs = pkgs .. ' ' .. site_vars(string.format('$(GLUON_SITE_PACKAGES) $(GLUON_SITE_PACKAGES_%s)', class)) + + pkgs = compact_list(split(pkgs)) + + class_cache[class] = pkgs + return pkgs end local function handle_target_pkgs(pkgs) - local packages = string.gmatch(pkgs, '%S+') - for pkg in packages do - lib.packages {pkg} + for _, pkg in ipairs(pkgs) do + if string.sub(pkg, 1, 1) == '-' then + lib.try_config('# CONFIG_PACKAGE_%s is not set', string.sub(pkg, 2)) + else + funcs.config_package(lib.config, pkg, 'y') + end end end lib.include('generic') - handle_target_pkgs(extra_packages) lib.include(target) - if lib.target_class ~= nil then - handle_target_pkgs(class_packages[lib.target_class]) - end - lib.check_devices() - if not lib.opkg then lib.config '# CONFIG_SIGNED_PACKAGES is not set' lib.config 'CONFIG_CLEAN_IPKG=y' lib.packages {'-opkg'} end + if #lib.devices > 0 then + handle_target_pkgs(lib.target_packages) - local default_pkgs = '' - for _, pkg in ipairs(lib.target_packages) do - default_pkgs = default_pkgs .. ' ' .. pkg + for _, dev in ipairs(lib.devices) do + local profile = dev.options.profile or dev.name - if string.sub(pkg, 1, 1) == '-' then - lib.try_config('# CONFIG_PACKAGE_%s is not set', string.sub(pkg, 2)) - else - funcs.config_package(lib.config, pkg, 'y') - end - end - - for _, dev in ipairs(lib.devices) do - local profile = dev.options.profile or dev.name - local device_pkgs = default_pkgs - - local function handle_pkg(pkg) - if string.sub(pkg, 1, 1) ~= '-' then - funcs.config_package(lib.config, pkg, 'm') + local device_pkgs = {} + local function handle_pkgs(pkgs) + for _, pkg in ipairs(pkgs) do + if string.sub(pkg, 1, 1) ~= '-' then + funcs.config_package(lib.config, pkg, 'm') + end + device_pkgs = append_to_list(device_pkgs, pkg) + end end - device_pkgs = device_pkgs .. ' ' .. pkg + + handle_pkgs(lib.target_packages) + handle_pkgs(class_packages(dev.options.class)) + handle_pkgs(dev.options.packages or {}) + handle_pkgs(site_packages(dev.image)) + + funcs.config_message(lib.config, string.format("unable to enable device '%s'", profile), + 'CONFIG_TARGET_DEVICE_%s_DEVICE_%s=y', openwrt_config_target, profile) + lib.config('CONFIG_TARGET_DEVICE_PACKAGES_%s_DEVICE_%s="%s"', + openwrt_config_target, profile, + table.concat(device_pkgs, ' ')) end + else + -- x86 fallback: no devices + local target_pkgs = {} local function handle_pkgs(pkgs) - local packages = string.gmatch(pkgs or '', '%S+') - for pkg in packages do - handle_pkg(pkg) + for _, pkg in ipairs(pkgs) do + target_pkgs = append_to_list(target_pkgs, pkg) end end - for _, pkg in ipairs(dev.options.packages or {}) do - handle_pkg(pkg) - end - handle_pkgs(site_packages(dev.image)) + -- Just hardcode the class for device-less targets to 'standard' + -- - this is x86 only at the moment, and it will have devices + -- in OpenWrt 19.07 + 1 as well + handle_pkgs(lib.target_packages) + handle_pkgs(class_packages('standard')) - handle_pkgs(class_packages[dev.options.class]) - - funcs.config_message(lib.config, string.format("unable to enable device '%s'", profile), - 'CONFIG_TARGET_DEVICE_%s_DEVICE_%s=y', openwrt_config_target, profile) - lib.config('CONFIG_TARGET_DEVICE_PACKAGES_%s_DEVICE_%s="%s"', - openwrt_config_target, profile, device_pkgs) + handle_target_pkgs(target_pkgs) end return lib diff --git a/scripts/target_lib.lua b/scripts/target_lib.lua index d669928f..0fc2d8b6 100644 --- a/scripts/target_lib.lua +++ b/scripts/target_lib.lua @@ -23,7 +23,6 @@ assert(env.GLUON_DEPRECATED) M.site_code = assert(assert(dofile('scripts/site_config.lua')('site.conf')).site_code) M.target_packages = {} -M.target_class = nil M.configs = {} M.devices = {} M.images = {} @@ -40,7 +39,7 @@ local default_options = { aliases = {}, manifest_aliases = {}, packages = {}, - class = "standard", + class = 'standard', deprecated = false, broken = false, } @@ -159,10 +158,6 @@ function F.config(...) M.configs[string.format(...)] = 2 end -function F.class(target_class) - M.target_class = target_class -end - function F.packages(pkgs) for _, pkg in ipairs(pkgs) do table.insert(M.target_packages, pkg) diff --git a/targets/x86-geode b/targets/x86-geode index 4f13f9a9..57519eb1 100644 --- a/targets/x86-geode +++ b/targets/x86-geode @@ -1,5 +1,3 @@ -class 'standard' - packages { 'kmod-3c59x', 'kmod-8139cp', diff --git a/targets/x86.inc b/targets/x86.inc index 461f4839..9536e805 100644 --- a/targets/x86.inc +++ b/targets/x86.inc @@ -1,8 +1,6 @@ config 'CONFIG_VDI_IMAGES=y' config 'CONFIG_VMDK_IMAGES=y' -class 'standard' - packages { 'kmod-3c59x', 'kmod-8139cp',