nils/gluon
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Backport parallel download locking support

Browse Source 
Fixes #733
This commit is contained in:
Matthias Schiffer 2016-04-24 09:11:20 +02:00
parent db32736d3b
commit 7f1c11cf7e
5 changed files with 190 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
patches/openwrt/0027-tools-add-tar-host-build-required-for-sort-name-which-was-only-added-in-the-latest-version.patch Normal file
View File

@ -0,0 +1,81 @@
From: Matthias Schiffer <mschiffer@universe-factory.net>
Date: Sun, 24 Apr 2016 08:44:30 +0200
Subject: tools: add tar host build, required for --sort=name which was only added in the latest version
Signed-off-by: Felix Fietkau <nbd@openwrt.org>
Backport of r46876
diff --git a/tools/Makefile b/tools/Makefile
index c6cded8..98ce8c5 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -81,6 +81,9 @@ endif
# dependency for tools which have patches directory
$(foreach tool, $(tools-y), $(if $(wildcard $(curdir)/$(tool)/patches),$(eval $(curdir)/$(tool)/compile += $(curdir)/patch/install)))
+$(foreach tool, $(tools-y), $(eval $(curdir)/$(tool)/compile += $(curdir)/tar/install))
+tools-y += tar
+
$(curdir)/builddirs := $(tools-y) $(tools-dep) $(tools-)
$(curdir)/builddirs-default := $(tools-y)
diff --git a/tools/tar/Makefile b/tools/tar/Makefile
new file mode 100644
index 0000000..975e783
--- /dev/null
+++ b/tools/tar/Makefile
@@ -0,0 +1,30 @@
+#
+# Copyright (C) 2015 OpenWrt.org
+#
+# This is free software, licensed under the GNU General Public License v2.
+# See /LICENSE for more information.
+#
+include $(TOPDIR)/rules.mk
+
+PKG_NAME:=tar
+PKG_VERSION:=1.28
+
+PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.xz
+PKG_SOURCE_URL:=@GNU/tar
+PKG_MD5SUM:=49b6306167724fe48f419a33a5beb857
+
+HOST_BUILD_PARALLEL := 1
+
+include $(INCLUDE_DIR)/host-build.mk
+
+HOSTCC := $(HOSTCC_NOCACHE)
+HOSTCXX := $(HOSTCXX_NOCACHE)
+
+HOST_CONFIGURE_ARGS += \
+ --without-posix-acls \
+ --without-selinux \
+ --without-xattrs \
+ --disable-acl \
+ --disable-nls
+
+$(eval $(call HostBuild))
diff --git a/tools/tar/patches/100-fix_xattr_disable.patch b/tools/tar/patches/100-fix_xattr_disable.patch
new file mode 100644
index 0000000..5735bd2
--- /dev/null
+++ b/tools/tar/patches/100-fix_xattr_disable.patch
@@ -0,0 +1,17 @@
+--- a/lib/xattr-at.c
++++ b/lib/xattr-at.c
+@@ -18,6 +18,8 @@
+
+ #include <config.h>
+
++#ifdef HAVE_XATTRS
++
+ #include "xattr-at.h"
+ #include "openat.h"
+
+@@ -108,3 +110,5 @@
+ #undef AT_FUNC_RESULT
+ #undef AT_FUNC_POST_FILE_PARAM_DECLS
+ #undef AT_FUNC_POST_FILE_ARGS
++
++#endif

31
patches/openwrt/0028-tools-compile-flock-before-everything-else.patch Normal file
View File

@ -0,0 +1,31 @@
From: Matthias Schiffer <mschiffer@universe-factory.net>
Date: Sun, 24 Apr 2016 08:44:47 +0200
Subject: tools: compile flock before everything else
Signed-off-by: Felix Fietkau <nbd@openwrt.org>
Backport of r48413
diff --git a/tools/Makefile b/tools/Makefile
index 98ce8c5..3402c08 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -26,7 +26,7 @@ endif
tools-$(BUILD_TOOLCHAIN) += gmp mpfr mpc libelf
tools-y += m4 libtool autoconf automake flex bison pkg-config sed mklibs
tools-y += sstrip make-ext4fs e2fsprogs mtd-utils mkimage
-tools-y += firmware-utils patch-image patch quilt yaffs2 flock padjffs2
+tools-y += firmware-utils patch-image patch quilt yaffs2 padjffs2
tools-y += mm-macros missing-macros xz cmake scons bc findutils gengetopt patchelf
tools-$(CONFIG_TARGET_orion_generic) += wrt350nv2-builder upslug2
tools-$(CONFIG_powerpc) += upx
@@ -84,6 +84,9 @@ $(foreach tool, $(tools-y), $(if $(wildcard $(curdir)/$(tool)/patches),$(eval $(
$(foreach tool, $(tools-y), $(eval $(curdir)/$(tool)/compile += $(curdir)/tar/install))
tools-y += tar
+$(curdir)/tar/compile := $(curdir)/flock/install
+tools-y += flock
+
$(curdir)/builddirs := $(tools-y) $(tools-dep) $(tools-)
$(curdir)/builddirs-default := $(tools-y)

32
patches/openwrt/0029-rules.mk-make-the-locked-template-available-even-if-flock-has-not-been-built-yet-fall-back-to-unlocked-shell-command.patch Normal file
View File

@ -0,0 +1,32 @@
From: Matthias Schiffer <mschiffer@universe-factory.net>
Date: Sun, 24 Apr 2016 08:45:44 +0200
Subject: rules.mk: make the locked template available even if flock has not been built yet (fall back to unlocked shell command)
Signed-off-by: Felix Fietkau <nbd@openwrt.org>
Backport of r48414
diff --git a/rules.mk b/rules.mk
index 9d0134d..ece5026 100644
--- a/rules.mk
+++ b/rules.mk
@@ -310,12 +310,16 @@ endef
# Execute commands under flock
# $(1) => The shell expression.
# $(2) => The lock name. If not given, the global lock will be used.
-define locked
+ifneq ($(wildcard $(STAGING_DIR_HOST)/bin/flock),)
+ define locked
SHELL= \
- $(STAGING_DIR_HOST)/bin/flock \
+ flock \
$(TMP_DIR)/.$(if $(2),$(strip $(2)),global).flock \
-c '$(subst ','\'',$(1))'
-endef
+ endef
+else
+ locked=$(1)
+endif
# Recursively copy paths into another directory, purge dangling
# symlinks before.

20
patches/openwrt/0030-build-add-locking-for-downloads-fixes-race-conditions-with-multiple-variants.patch Normal file
View File

@ -0,0 +1,20 @@
From: Matthias Schiffer <mschiffer@universe-factory.net>
Date: Sun, 24 Apr 2016 08:48:45 +0200
Subject: build: add locking for downloads (fixes race conditions with multiple variants)
Signed-off-by: Felix Fietkau <nbd@openwrt.org>
Backport of r48416
diff --git a/include/download.mk b/include/download.mk
index e518cce..9176b11 100644
--- a/include/download.mk
+++ b/include/download.mk
@@ -182,6 +182,6 @@ define Download
$(DL_DIR)/$(FILE):
mkdir -p $(DL_DIR)
- $(if $(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/unknown))
+ $(call locked,$(if $(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/$(call dl_method,$(URL),$(PROTO))),$(DownloadMethod/unknown)),$(FILE))
endef

26
patches/openwrt/0031-download.mk-fix-packed-checkout-mirroring-support.patch Normal file
View File

@ -0,0 +1,26 @@
From: Matthias Schiffer <mschiffer@universe-factory.net>
Date: Sun, 24 Apr 2016 08:49:27 +0200
Subject: download.mk: fix packed checkout mirroring support
Changeset r48416 broke the downloading of mirrored, packed scm checkouts.
Fix this by removing the "@" sign in front of the download command which is
now executed as part of a larger shell command under flock.
Signed-off-by: Jo-Philipp Wich <jow@openwrt.org>
Backport of r48733
diff --git a/include/download.mk b/include/download.mk
index 9176b11..130bbc5 100644
--- a/include/download.mk
+++ b/include/download.mk
@@ -48,7 +48,7 @@ define DownloadMethod/default
endef
define wrap_mirror
- $(if $(if $(MIRROR),$(filter-out x,$(MIRROR_MD5SUM))),@$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "$(MIRROR_MD5SUM)" "" || ( $(1) ),$(1))
+$(if $(if $(MIRROR),$(filter-out x,$(MIRROR_MD5SUM))),$(SCRIPT_DIR)/download.pl "$(DL_DIR)" "$(FILE)" "$(MIRROR_MD5SUM)" "" || ( $(1) ),$(1))
endef
define DownloadMethod/cvs