From bac4f6a8f49f62d51adb305338b5bab423685808 Mon Sep 17 00:00:00 2001 From: Tobias Dreyer Date: Mon, 21 May 2012 16:08:03 +0200 Subject: [PATCH 1/6] filelist2content: new utility to convert a filelist to new CONTENT file output --- Makefile | 2 +- src/filelist2content.sh.in | 160 +++++++++++++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 1 deletion(-) create mode 100644 src/filelist2content.sh.in diff --git a/Makefile b/Makefile index dd26fa0..7441226 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ PROGRAMS_PERL=beefind.pl HELPER_BEE_SHELL=bee-init bee-check bee-remove bee-install bee-list bee-query bee-download bee-update HELPER_BEE_C=bee-dep -HELPER_SHELL=compat-filesfile2contentfile +HELPER_SHELL=compat-filesfile2contentfile filelist2content LIBRARY_SHELL=beelib.config.sh diff --git a/src/filelist2content.sh.in b/src/filelist2content.sh.in new file mode 100644 index 0000000..4ab710a --- /dev/null +++ b/src/filelist2content.sh.in @@ -0,0 +1,160 @@ +#!/bin/bash +# +# filelist2content - create content file output to specified files +# +# Copyright (C) 2009-2012 +# Tobias Dreyer +# Marius Tolzmann +# and other bee developers +# +# This file is part of bee. +# +# bee is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +: ${BEE_GETOPT:=@BINDIR@/beegetopt} + +function get_format_string() { + local format_string + + format_string="type=\"%F\"" + format_string="${format_string} mode=%f" + format_string="${format_string} uid=%u" + format_string="${format_string} user=%U" + format_string="${format_string} gid=%g" + format_string="${format_string} group=%G" + format_string="${format_string} size=%s" + format_string="${format_string} mtime=%Y" + format_string="${format_string} nlink=%h" + format_string="${format_string} inode=%i" + format_string="${format_string} major=%t" + format_string="${format_string} minor=%T" + format_string="${format_string} device=%d" + + echo ${format_string} +} + +function get_type() { + local type=${1} + + if [ "${type}" = "symbolic" ] ; then + type="symlink" + fi + + echo ${type} +} + +function do_f2c() { + local filelist=${1} + local root=${2} + declare -A hardlinks + + if [ ! -r "${filelist}" ] ; then + print_error "internal error: failed to read filelist" + exit 1 + fi + + while read filename ; do + data=$(stat --format "$(get_format_string)" ${root}${filename}) + + if [ "$?" -gt 0 ] ; then + echo >&2 "**ERROR** stat failed" + exit 1 + fi + + eval ${data} + file=${filename} + + if [ "$?" -gt 0 ] ; then + echo >&2 "**ERROR** eval failed" + exit 1 + fi + + type=$(get_type ${type}) + + if [ ${nlink} -gt 1 ] ; then + key="${inode}-${device}" + if [ -z "${hardlinks[${key}]}" ] ; then + hardlinks[${key}]=${filename} + else + type=hardlink + file="${file}//${hardlinks[${key}]}" + fi + fi + + echo -n "type=${type}" + + printf ":mode=0%o" "0x${mode}" + printf ":access=0%o" $(( 0x${mode} & 07777 )) + + echo -n ":uid=${uid}" + echo -n ":user=${user}" + echo -n ":gid=${gid}" + echo -n ":group=${group}" + echo -n ":size=${size}" + echo -n ":mtime=${mtime}" + echo -n ":nlink=${nlink}" + + if [ "${type}" = "regular" -o "${type}" = "hardlink" ] ; then + md5=$(md5sum ${root}${filename}) + + if [ "$?" -gt 0 ] ; then + echo >&2 "**ERROR** md5sum failed" + exit 1 + fi + + echo -n ":md5=${md5%% *}" + elif [ "${type}" = "symlink" ] ; then + file="${file}//$(readlink ${file})" + elif [ "${type}" = "block" -o "${type}" = "char" ] ; then + echo -n ":major=${major}" + echo -n ":minor=${minor}" + fi + + echo -n ":file=${file}" + echo "" + + done < "${filelist}" +} + +############################################################################### +############################################################################### +############################################################################### + +options=$(${BEE_GETOPT} --name filelist2content \ + --option root/r= \ + -- "${@}") + +if [ $? != 0 ] ; then + echo >&2 "**ERROR** beegetopt failed" + exit 1 +fi + +eval set -- "${options}" + +declare OPT_ROOT= + +while true ; do + case "${1}" in + --root) + OPT_ROOT="${2}" + shift 2 + ;; + --) + shift + break + ;; + esac +done + +do_f2c <(cat ${@}) "${OPT_ROOT}" From f49a3315227e21f3a9040b72098566ad86c07dbb Mon Sep 17 00:00:00 2001 From: Tobias Dreyer Date: Tue, 22 May 2012 08:57:36 +0200 Subject: [PATCH 2/6] content2filelist: new utility to extract files from a CONTENT file --- Makefile | 2 +- src/content2filelist.sh.in | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 1 deletion(-) create mode 100644 src/content2filelist.sh.in diff --git a/Makefile b/Makefile index 7441226..d99bf71 100644 --- a/Makefile +++ b/Makefile @@ -60,7 +60,7 @@ PROGRAMS_PERL=beefind.pl HELPER_BEE_SHELL=bee-init bee-check bee-remove bee-install bee-list bee-query bee-download bee-update HELPER_BEE_C=bee-dep -HELPER_SHELL=compat-filesfile2contentfile filelist2content +HELPER_SHELL=compat-filesfile2contentfile filelist2content content2filelist LIBRARY_SHELL=beelib.config.sh diff --git a/src/content2filelist.sh.in b/src/content2filelist.sh.in new file mode 100644 index 0000000..9c08c4a --- /dev/null +++ b/src/content2filelist.sh.in @@ -0,0 +1,44 @@ +#!/bin/bash +# +# content2filelist - extract files from content file +# +# Copyright (C) 2009-2012 +# Tobias Dreyer +# Marius Tolzmann +# and other bee developers +# +# This file is part of bee. +# +# bee is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +: ${BEESEP=@BINDIR@/beesep} + +while read line ; do + unset type + data=$(${BEESEP} ${line}) + + if [ "$?" -gt 0 ] ; then + echo >&2 "**ERROR** INVALID CONTENT: ${line}" + exit 1 + fi + + eval ${data} + + if [ "$?" -gt 0 -o -z "${type}" ] ; then + echo >&2 "**ERROR** UNPARSABLE CONTENT: ${line}" + exit 1 + fi + + echo ${file%//*} +done < <(cat ${@}) From 78cba40f8386024b9117245d9b4a6c8dfe2a5eca Mon Sep 17 00:00:00 2001 From: Tobias Dreyer Date: Tue, 22 May 2012 09:29:48 +0200 Subject: [PATCH 3/6] beefind: some KISSing now beefind just scans a directory and prints a file list and not the files metadata --- src/beefind.sh.in | 121 ++++++---------------------------------------- 1 file changed, 16 insertions(+), 105 deletions(-) diff --git a/src/beefind.sh.in b/src/beefind.sh.in index 9854067..ce39c09 100644 --- a/src/beefind.sh.in +++ b/src/beefind.sh.in @@ -1,6 +1,6 @@ #!/bin/bash # -# beefind - scan directory and print files and metadata +# beefind - scan directory and print files # # Copyright (C) 2009-2012 # Tobias Dreyer @@ -31,6 +31,7 @@ BEE_VERSION="@BEE_VERSION@" : ${BEEVERSION:=${BEE_BINDIR}/beeversion} : ${BEESEP=${BEE_BINDIR}/beesep} +: ${BEEGETOPT:=${BEE_BINDIR}/beegetopt} function usage() { cat <<-EOF @@ -38,70 +39,22 @@ function usage() { by Tobias Dreyer and Marius Tolzmann <{dreyer,tolzmann}@molgen.mpg.de> Max Planck Institute for Molecular Genetics Berlin Dahlem - Usage: beefind [options] + Usage: beefind [options] Options: - -c | --cutroot strip from displayed files - -e | --exclude ignore files matching - -s | --skiplist ignore files matching pattern described in + -c, --cutroot strip from printed files + -e, --exclude ignore files matching + -s, --skiplist ignore files matching pattern described in - -o | --output output is redirected into instead of - -d | --dump-files read and just display files; must contain - beefind output + -o, --output output is redirected into instead of - -h | --help display this help + -h, --help display this help EOF } -function dump_files() { - local infile=${1} - - if [ ! -r "${infile}" ] ; then - print_warning "cannot read file '${infile}'" - exit 1 - fi - - for line in $(cat ${infile}) ; do - eval $(${BEESEP} ${line}) - - echo ${file%//*} - done -} - -function get_md5() { - local file=${1} - - if [ -L "${file}" ] ; then - md5='link' - elif [ -d "${file}" ] ; then - md5='directory' - elif [ -f "${file}" ] ; then - md5=$(md5sum ${file}) - md5=${md5%% *} - if [ "$?" != 0 -o -z "${md5}" ] ; then - print_warning "failed to create md5 sum for ${file}" - md5='#MD5ERROR#' - fi - elif [ -p "${file}" ] ; then - md5='pipe' - elif [ -S "${file}" ] ; then - md5='socket' - elif [ -b "${file}" ] ; then - md5='block' - elif [ -c "${file}" ] ; then - md5='char' - elif [ -S "${file}" ] ; then - md5='socket' - else - md5='#MD5UNKNOWN#' - fi - - echo ${md5} -} - function bee_find() { - local options=$(${BEE_BINDIR}/beegetopt --name beefind \ + local options=$(${BEEGETOPT} --name beefind \ --option exclude/e= \ --option skiplist/s= \ -- "${@}") @@ -114,7 +67,7 @@ function bee_find() { eval set -- "${options}" declare -a OPT_EXCLUDE - declare -a OPT_SKIPLSIT + declare -a OPT_SKIPLIST while true ; do case "${1}" in @@ -134,7 +87,7 @@ function bee_find() { done if [ -z "${1}" ] ; then - print_warning "no directory to scan specified" + usage exit 1 fi @@ -151,7 +104,7 @@ function bee_find() { } function do_beefind() { - local options=$(${BEE_BINDIR}/beegetopt --name beefind \ + local options=$(${BEEGETOPT} --name beefind \ --option cutroot/c= \ -- "${@}") @@ -177,44 +130,17 @@ function do_beefind() { esac done - for f in $(cat <(bee_find ${@})) ; do - md5=$(get_md5 ${f}) - - data=( $(stat -c "%05a %h %u %g %s %Y" ${f}) ) - if [ "$?" != 0 -o -z "${data}" ] ; then - print_warning "can't stat ${f}" - exit 1 - fi - - echo -n "md5=${md5}:" - echo -n "omode=${data[0]}:" - echo -n "nlink=${data[1]}:" - echo -n "uid=${data[2]}:" - echo -n "gid=${data[3]}:" - echo -n "size=${data[4]}:" - echo -n "mtime=${data[5]}:" - - filename=${f} - if [ -n "${OPT_CUTROOT}" ] ; then - filename=${f#${OPT_CUTROOT}} - fi - echo -n "file=${filename}" - - if [ "${md5}" = "link" ] ; then - echo -n "//$(readlink ${f})" - fi - - echo "" - done + while read line ; do + echo ${line#${OPT_CUTROOT}} + done < <(bee_find ${@}) } ############################################################################### ############################################################################### ############################################################################### -options=$(${BEE_BINDIR}/beegetopt --name beefind \ +options=$(${BEEGETOPT} --name beefind \ --option output/o= \ - --option dump-files/d= \ --option help/h \ -- "${@}") @@ -226,7 +152,6 @@ fi eval set -- "${options}" declare OPT_OUTPUT= -declare OPT_DUMPFILES= while true ; do case "${1}" in @@ -234,10 +159,6 @@ while true ; do OPT_OUTPUT="${2}" shift 2 ;; - --dump-files) - OPT_DUMPFILES="${2}" - shift 2 - ;; --help) usage exit 0 @@ -256,14 +177,4 @@ if [ -n "${OPT_OUTPUT}" ] ; then fi fi -if [ -n "${OPT_DUMPFILES}" ] ; then - if [ ! -r "${OPT_DUMPFILES}" ] ; then - print_warning "cannot read file '${OPT_DUMPFILES}'" - exit 1 - fi - - dump_files ${OPT_DUMPFILES} - exit 0 -fi - do_beefind ${@} From 46f9022609f00ca2f00e544ae493a663610012cc Mon Sep 17 00:00:00 2001 From: Tobias Dreyer Date: Tue, 22 May 2012 10:09:59 +0200 Subject: [PATCH 4/6] beefind: bugfix: cutting rootdir before excluding files --- src/beefind.sh.in | 64 +++++++++++++++-------------------------------- 1 file changed, 20 insertions(+), 44 deletions(-) diff --git a/src/beefind.sh.in b/src/beefind.sh.in index ce39c09..6275d7b 100644 --- a/src/beefind.sh.in +++ b/src/beefind.sh.in @@ -53,8 +53,9 @@ function usage() { EOF } -function bee_find() { +function do_beefind() { local options=$(${BEEGETOPT} --name beefind \ + --option cutroot/c \ --option exclude/e= \ --option skiplist/s= \ -- "${@}") @@ -66,11 +67,16 @@ function bee_find() { eval set -- "${options}" + declare find_format="%p\n" declare -a OPT_EXCLUDE declare -a OPT_SKIPLIST while true ; do case "${1}" in + --cutroot) + find_format="/%P\n" + shift 1 + ;; --exclude) OPT_EXCLUDE=( ${OPT_EXCLUDE:+${OPT_EXCLUDE[@]}} "${2}" ) shift 2 @@ -86,53 +92,23 @@ function bee_find() { esac done - if [ -z "${1}" ] ; then + if [ -z "${@}" ] ; then usage exit 1 fi - find ${1} -mindepth 1 | \ - grep \ - --extended-regexp \ - --invert-match \ - --file=<(if [ "${#OPT_SKIPLIST[@]}" -gt 0 ] ; then - grep --invert-match --regexp="^ *$" ${OPT_SKIPLIST[@]} - fi) \ - --file=<(for p in ${OPT_EXCLUDE[@]} ; do - echo ${p} - done) -} - -function do_beefind() { - local options=$(${BEEGETOPT} --name beefind \ - --option cutroot/c= \ - -- "${@}") - - if [ $? != 0 ] ; then - usage - exit 1 - fi - - eval set -- "${options}" - - declare OPT_CUTROOT= - - while true ; do - case "${1}" in - --cutroot) - OPT_CUTROOT="${2}" - shift 2 - ;; - --) - shift - break - ;; - esac - done - - while read line ; do - echo ${line#${OPT_CUTROOT}} - done < <(bee_find ${@}) + dirs=( "${@}" ) + + grep \ + --extended-regexp \ + --invert-match \ + --file=<(if [ "${#OPT_SKIPLIST[@]}" -gt 0 ] ; then + grep --invert-match --regexp="^ *$" ${OPT_SKIPLIST[@]} + fi) \ + --file=<(for p in ${OPT_EXCLUDE[@]} ; do + echo ${p} + done) \ + <(find ${dirs[@]} -mindepth 1 -xdev -printf ${find_format}) } ############################################################################### From 62aaac0cba8dc19f8002361f541fc97760351e27 Mon Sep 17 00:00:00 2001 From: Tobias Dreyer Date: Tue, 22 May 2012 10:54:52 +0200 Subject: [PATCH 5/6] beefind: rename option '--skiplist' to '--exclude-list' to easily integrate the new beefind --- src/beefind.sh.in | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/beefind.sh.in b/src/beefind.sh.in index 6275d7b..59faf09 100644 --- a/src/beefind.sh.in +++ b/src/beefind.sh.in @@ -42,13 +42,13 @@ function usage() { Usage: beefind [options] Options: - -c, --cutroot strip from printed files - -e, --exclude ignore files matching - -s, --skiplist ignore files matching pattern described in + -c, --cutroot strip from printed files + -e, --exclude ignore files matching + -s, --exclude-list ignore files matching pattern described in - -o, --output output is redirected into instead of + -o, --output output is redirected into instead of - -h, --help display this help + -h, --help display this help EOF } @@ -57,7 +57,7 @@ function do_beefind() { local options=$(${BEEGETOPT} --name beefind \ --option cutroot/c \ --option exclude/e= \ - --option skiplist/s= \ + --option exclude-list/E= \ -- "${@}") if [ $? != 0 ] ; then @@ -69,7 +69,7 @@ function do_beefind() { declare find_format="%p\n" declare -a OPT_EXCLUDE - declare -a OPT_SKIPLIST + declare -a OPT_EXCLUDELIST while true ; do case "${1}" in @@ -78,11 +78,11 @@ function do_beefind() { shift 1 ;; --exclude) - OPT_EXCLUDE=( ${OPT_EXCLUDE:+${OPT_EXCLUDE[@]}} "${2}" ) + OPT_EXCLUDE=( "${OPT_EXCLUDE[@]}" "${2}" ) shift 2 ;; - --skiplist) - OPT_SKIPLIST=( ${OPT_SKIPLIST:+${OPT_SKIPLIST[@]}} "${2}" ) + --exclude-list) + OPT_EXCLUDELIST=( "${OPT_EXCLUDELIST[@]}" "${2}" ) shift 2 ;; --) @@ -102,8 +102,8 @@ function do_beefind() { grep \ --extended-regexp \ --invert-match \ - --file=<(if [ "${#OPT_SKIPLIST[@]}" -gt 0 ] ; then - grep --invert-match --regexp="^ *$" ${OPT_SKIPLIST[@]} + --file=<(if [ "${#OPT_EXCLUDELIST[@]}" -gt 0 ] ; then + grep --invert-match --regexp="^ *$" ${OPT_EXCLUDELIST[@]} fi) \ --file=<(for p in ${OPT_EXCLUDE[@]} ; do echo ${p} From 574222436cacba0c2def01a1ee43e8bbc6bd706f Mon Sep 17 00:00:00 2001 From: Tobias Dreyer Date: Tue, 22 May 2012 12:11:07 +0200 Subject: [PATCH 6/6] beesh: integrate new beefind --- src/beesh.sh.in | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/beesh.sh.in b/src/beesh.sh.in index d5c74a2..3c35ae9 100644 --- a/src/beesh.sh.in +++ b/src/beesh.sh.in @@ -39,6 +39,8 @@ VERSION=${BEE_VERSION} : ${BEE_LIBEXECDIR:=@LIBEXECDIR@} : ${BEE_BINDIR:=@BINDIR@} +: ${BEEFIND:=${BEE_BINDIR}/beefind} + # load libs . ${BEE_LIBEXECDIR}/bee/beelib.config.sh @@ -432,21 +434,23 @@ function bee_crosscheck() { # $EXCLUDE is read from .bee file # $BEE_SKIPLIST is found in $BEEFAULTS function bee_pkg_pack() { - beefind.pl --exclude='^/FILES$' \ - --excludelist=<( + ${BEEFIND} --exclude='^/CONTENT$' \ + --exclude-list=<( if [ -n "${BEE_SKIPLIST}" ] ; then cat ${BEE_SKIPLIST} fi for pattern in "${EXCLUDE[@]}" "${BEE_AUTO_EXCLUDE[@]}" ; do echo "${pattern}" done ) \ - --cutroot=${D} ${D} > ${D}/FILES 2>/dev/null + --cutroot \ + ${D} | \ + ${BEE_LIBEXECDIR}/bee/filelist2content --root ${D} > ${D}/CONTENT DUMP=${BEE_TMP_TMPDIR}/bee.$$.dump - beefind.pl --dump ${D}/FILES | sed -e "s,^,${D}," - > ${DUMP} + ${BEE_LIBEXECDIR}/bee/content2filelist ${D}/CONTENT | sed -e "s,^,${D}," - > ${DUMP} - if [ ! -s "${D}/FILES" ]; then + if [ ! -s "${D}/CONTENT" ]; then print_error "ERROR: empty image directory" exit 1 fi @@ -479,11 +483,11 @@ function bee_pkg_pack() { --sparse \ --absolute-names \ --no-recursion \ - --transform="s,^/FILES$,FILES," \ + --transform="s,^/CONTENT$,CONTENT," \ --transform="s,^/BUILD$,BUILD," \ --transform="s,^/META$,META," \ --transform="s,^/PATCHES,PATCHES," \ - ${D}/{FILES,BUILD,META} \ + ${D}/{CONTENT,BUILD,META} \ ${bee_PATCHFILES:+${D}/PATCHES} \ ${bee_PATCHFILES:+${D}/PATCHES/*}