Skip to content

Commit

Permalink
kbuild: give up untracked files for source package builds
Browse files Browse the repository at this point in the history
When the source tree is dirty and contains untracked files, package
builds may fail, for example, when a broken symlink exists, a file
path contains whitespaces, etc.

Since commit 05e96e9 ("kbuild: use git-archive for source package
creation"), the source tarball only contains committed files because
it is created by 'git archive'. scripts/package/gen-diff-patch tries
to address the diff from HEAD, but including untracked files by the
hand-crafted script introduces more complexity. I wrote a patch [1] to
make it work in most cases, but still wonder if this is what we should
aim for.

To simplify the code, this patch just gives up untracked files. Going
forward, it is your responsibility to do 'git add' for what you want in
the source package. The script shows a warning just in case you forgot
to do so. It should be checked only when building source packages.

[1]: https://lore.kernel.org/all/CAK7LNAShbZ56gSh9PrbLnBDYKnjtTkHMoCXeGrhcxMvqXGq9=g@mail.gmail.com/2-0001-kbuild-make-package-builds-more-robust.patch

Fixes: 05e96e9 ("kbuild: use git-archive for source package creation")
Signed-off-by: Masahiro Yamada <masahiroy@kernel.org>
Reviewed-by: Nicolas Schier <nicolas@fjasle.eu>
  • Loading branch information
Masahiro Yamada committed Apr 10, 2023
1 parent dcc11ac commit aa7d233
Show file tree
Hide file tree
Showing 4 changed files with 90 additions and 89 deletions.
3 changes: 2 additions & 1 deletion scripts/Makefile.package
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ binrpm-pkg:
$(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec

quiet_cmd_debianize = GEN $@
cmd_debianize = $(srctree)/scripts/package/mkdebian
cmd_debianize = $(srctree)/scripts/package/mkdebian $(mkdebian-opts)

debian: FORCE
$(call cmd,debianize)
Expand All @@ -103,6 +103,7 @@ PHONY += debian-orig
debian-orig: private source = $(shell dpkg-parsechangelog -S Source)
debian-orig: private version = $(shell dpkg-parsechangelog -S Version | sed 's/-[^-]*$$//')
debian-orig: private orig-name = $(source)_$(version).orig.tar.gz
debian-orig: mkdebian-opts = --need-source
debian-orig: linux.tar.gz debian
$(Q)if [ "$(df --output=target .. 2>/dev/null)" = "$(df --output=target $< 2>/dev/null)" ]; then \
ln -f $< ../$(orig-name); \
Expand Down
62 changes: 27 additions & 35 deletions scripts/package/gen-diff-patch
Original file line number Diff line number Diff line change
@@ -1,44 +1,36 @@
#!/bin/sh
# SPDX-License-Identifier: GPL-2.0-only

diff_patch="${1}"
untracked_patch="${2}"
srctree=$(dirname $0)/../..
diff_patch=$1

rm -f ${diff_patch} ${untracked_patch}
mkdir -p "$(dirname "${diff_patch}")"

if ! ${srctree}/scripts/check-git; then
exit
fi

mkdir -p "$(dirname ${diff_patch})" "$(dirname ${untracked_patch})"
git -C "${srctree:-.}" diff HEAD > "${diff_patch}"

git -C "${srctree}" diff HEAD > "${diff_patch}"

if [ ! -s "${diff_patch}" ]; then
rm -f "${diff_patch}"
if [ ! -s "${diff_patch}" ] ||
[ -z "$(git -C "${srctree:-.}" ls-files --other --exclude-standard | head -n1)" ]; then
exit
fi

git -C ${srctree} status --porcelain --untracked-files=all |
while read stat path
do
if [ "${stat}" = '??' ]; then

if ! diff -u /dev/null "${srctree}/${path}" > .tmp_diff &&
! head -n1 .tmp_diff | grep -q "Binary files"; then
{
echo "--- /dev/null"
echo "+++ linux/$path"
cat .tmp_diff | tail -n +3
} >> ${untracked_patch}
fi
fi
done

rm -f .tmp_diff

if [ ! -s "${diff_patch}" ]; then
rm -f "${diff_patch}"
exit
fi
# The source tarball, which is generated by 'git archive', contains everything
# you committed in the repository. If you have local diff ('git diff HEAD'),
# it will go into ${diff_patch}. If untracked files are remaining, the resulting
# source package may not be correct.
#
# Examples:
# - You modified a source file to add #include "new-header.h"
# but forgot to add new-header.h
# - You modified a Makefile to add 'obj-$(CONFIG_FOO) += new-dirver.o'
# but you forgot to add new-driver.c
#
# You need to commit them, or at least stage them by 'git add'.
#
# This script does not take care of untracked files because doing so would
# introduce additional complexity. Instead, print a warning message here if
# untracked files are found.
# If all untracked files are just garbage, you can ignore this warning.
echo >&2 "============================ WARNING ============================"
echo >&2 "Your working tree has diff from HEAD, and also untracked file(s)."
echo >&2 "Please make sure you did 'git add' for all new files you need in"
echo >&2 "the source package."
echo >&2 "================================================================="
103 changes: 59 additions & 44 deletions scripts/package/mkdebian
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,66 @@ set_debarch() {
fi
}

# Create debian/source/ if it is a source package build
gen_source ()
{
mkdir -p debian/source

echo "3.0 (quilt)" > debian/source/format

{
echo "diff-ignore"
echo "extend-diff-ignore = .*"
} > debian/source/local-options

# Add .config as a patch
mkdir -p debian/patches
{
echo "Subject: Add .config"
echo "Author: ${maintainer}"
echo
echo "--- /dev/null"
echo "+++ linux/.config"
diff -u /dev/null "${KCONFIG_CONFIG}" | tail -n +3
} > debian/patches/config.patch
echo config.patch > debian/patches/series

"${srctree}/scripts/package/gen-diff-patch" debian/patches/diff.patch
if [ -s debian/patches/diff.patch ]; then
sed -i "
1iSubject: Add local diff
1iAuthor: ${maintainer}
1i
" debian/patches/diff.patch

echo diff.patch >> debian/patches/series
else
rm -f debian/patches/diff.patch
fi
}

rm -rf debian
mkdir debian

email=${DEBEMAIL-$EMAIL}

# use email string directly if it contains <email>
if echo "${email}" | grep -q '<.*>'; then
maintainer=${email}
else
# or construct the maintainer string
user=${KBUILD_BUILD_USER-$(id -nu)}
name=${DEBFULLNAME-${user}}
if [ -z "${email}" ]; then
buildhost=${KBUILD_BUILD_HOST-$(hostname -f 2>/dev/null || hostname)}
email="${user}@${buildhost}"
fi
maintainer="${name} <${email}>"
fi

if [ "$1" = --need-source ]; then
gen_source
fi

# Some variables and settings used throughout the script
version=$KERNELRELEASE
Expand All @@ -104,22 +163,6 @@ fi
debarch=
set_debarch

email=${DEBEMAIL-$EMAIL}

# use email string directly if it contains <email>
if echo $email | grep -q '<.*>'; then
maintainer=$email
else
# or construct the maintainer string
user=${KBUILD_BUILD_USER-$(id -nu)}
name=${DEBFULLNAME-$user}
if [ -z "$email" ]; then
buildhost=${KBUILD_BUILD_HOST-$(hostname -f 2>/dev/null || hostname)}
email="$user@$buildhost"
fi
maintainer="$name <$email>"
fi

# Try to determine distribution
if [ -n "$KDEB_CHANGELOG_DIST" ]; then
distribution=$KDEB_CHANGELOG_DIST
Expand All @@ -132,34 +175,6 @@ else
echo >&2 "Install lsb-release or set \$KDEB_CHANGELOG_DIST explicitly"
fi

mkdir -p debian/source/
echo "3.0 (quilt)" > debian/source/format

{
echo "diff-ignore"
echo "extend-diff-ignore = .*"
} > debian/source/local-options

# Add .config as a patch
mkdir -p debian/patches
{
echo "Subject: Add .config"
echo "Author: ${maintainer}"
echo
echo "--- /dev/null"
echo "+++ linux/.config"
diff -u /dev/null "${KCONFIG_CONFIG}" | tail -n +3
} > debian/patches/config
echo config > debian/patches/series

$(dirname $0)/gen-diff-patch debian/patches/diff.patch debian/patches/untracked.patch
if [ -f debian/patches/diff.patch ]; then
echo diff.patch >> debian/patches/series
fi
if [ -f debian/patches/untracked.patch ]; then
echo untracked.patch >> debian/patches/series
fi

echo $debarch > debian/arch
extra_build_depends=", $(if_enabled_echo CONFIG_UNWINDER_ORC libelf-dev:native)"
extra_build_depends="$extra_build_depends, $(if_enabled_echo CONFIG_SYSTEM_TRUSTED_KEYRING libssl-dev:native)"
Expand Down
11 changes: 2 additions & 9 deletions scripts/package/mkspec
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ else
mkdir -p rpmbuild/SOURCES
cp linux.tar.gz rpmbuild/SOURCES
cp "${KCONFIG_CONFIG}" rpmbuild/SOURCES/config
$(dirname $0)/gen-diff-patch rpmbuild/SOURCES/diff.patch rpmbuild/SOURCES/untracked.patch
touch rpmbuild/SOURCES/diff.patch rpmbuild/SOURCES/untracked.patch
"${srctree}/scripts/package/gen-diff-patch" rpmbuild/SOURCES/diff.patch
fi

if grep -q CONFIG_MODULES=y include/config/auto.conf; then
Expand Down Expand Up @@ -56,7 +55,6 @@ sed -e '/^DEL/d' -e 's/^\t*//' <<EOF
$S Source0: linux.tar.gz
$S Source1: config
$S Source2: diff.patch
$S Source3: untracked.patch
Provides: $PROVIDES
$S BuildRequires: bc binutils bison dwarves
$S BuildRequires: (elfutils-libelf-devel or libelf-devel) flex
Expand Down Expand Up @@ -94,12 +92,7 @@ $S$M
$S %prep
$S %setup -q -n linux
$S cp %{SOURCE1} .config
$S if [ -s %{SOURCE2} ]; then
$S patch -p1 < %{SOURCE2}
$S fi
$S if [ -s %{SOURCE3} ]; then
$S patch -p1 < %{SOURCE3}
$S fi
$S patch -p1 < %{SOURCE2}
$S
$S %build
$S $MAKE %{?_smp_mflags} KERNELRELEASE=$KERNELRELEASE KBUILD_BUILD_VERSION=%{release}
Expand Down

0 comments on commit aa7d233

Please sign in to comment.