Skip to content

Instantly share code, notes, and snippets.

@satmandu

satmandu/chromeos_docker.sh

Last active March 16, 2021 20:42
Show Gist options
  • Save satmandu/791dd6e982d365dda6bbba5c332f0f73 to your computer and use it in GitHub Desktop.
Save satmandu/791dd6e982d365dda6bbba5c332f0f73 to your computer and use it in GitHub Desktop.
Download ZIP
Docker buildx based package builder for Chromebrew on x86_64, i686, & armv7l. (armv7l only tested on Raspberry PI) Put both files in the same folder.
Raw
chromeos_docker.sh
#!/bin/bash
# chromeos_docker.sh
# Usage:
# chromeos_docker.sh recoveryfileurl name milestone arch
# or if image.bin already exists this works too:
# chromeos_docker.sh dummy name milestone arch
# (Default is not to delete the image after download.)
# e.g.
# Example for x86_64:
# chromeos_docker.sh https://dl.google.com/dl/edgedl/chromeos/recovery/chromeos_13505.73.0_nocturne_recovery_stable-channel_mp.bin.zip nocturne 87 x86_64
# Example for armv7l:
# chromeos_docker.sh https://dl.google.com/dl/edgedl/chromeos/recovery/chromeos_13505.73.0_veyron-fievel_recovery_stable-channel_fievel-mp.bin.zip fievel 87 armv7l
# Examples for i686:
# chromeos_docker.sh https://dl.google.com/dl/edgedl/chromeos/recovery/chromeos_9334.72.0_x86-alex-he_recovery_stable-channel_alex-mp-v4.bin.zip alex 58 i686
# chromeos_docker.sh https://dl.google.com/dl/edgedl/chromeos/recovery/chromeos_9334.72.0_x86-zgb-he_recovery_stable-channel_zgb-mp-v3.bin.zip zgb 58 i686
# Recovery file update links are here: https://cros-updates-serving.appspot.com/
# Note that you may need further steps if you are on an x86_64 machine.
# See https://www.stereolabs.com/docs/docker/building-arm-container-on-x86/
#
# Note also you should have a registry setup.
# Instructions here: https://docs.docker.com/registry/deploying/
# If you setup your registry with an ssl cert, you may have fewer problems.
# You can set the registry URL with the REPOSITORY env variable.
# e.g. export REPOSITORY="dockerserver:5000"
imageurl=${1}
name=${2}
milestone=${3}
ARCH=${4}
outdir=$(pwd)
REPOSITORY="${REPOSITORY:-"127.0.0.1:5000"}"
echo " image url:${imageurl}"
echo " name: ${name}"
echo " milestone: ${milestone}"
echo " ARCH: ${ARCH}"
echo " REPOSITORY: ${REPOSITORY}"
echo "output root: ${outdir}"
userid=$(id -un)
tmpdir=$(mktemp -d docker_XXXX -p "$(pwd)")
function abspath {
echo $(cd "$1" && pwd)
}
countdown()
(
IFS=:
set -- $*
secs=$(( ${1#0} * 3600 + ${2#0} * 60 + ${3#0} ))
while [ $secs -gt 0 ]
do
sleep 1 &
printf "\r%02d:%02d:%02d" $((secs/3600)) $(( (secs/60)%60)) $((secs%60))
secs=$(( $secs - 1 ))
wait
done
echo
)
get_arch () {
if [[ ! -f "$ARCH/image.bin" ]] ; then
curl --retry 3 -Lf "$imageurl" -o $ARCH/image.bin.zip || ( echo "Download failed" && kill $$ )
unzip -p $ARCH/image.bin.zip > $ARCH/image.bin && rm $ARCH/image.bin.zip
fi
sudo kpartx -d $ARCH/image.bin
rootpart=$(sudo kpartx -v -a $ARCH/image.bin | grep 'p3\b' | awk '{print $3}')
[[ -z "$rootpart" ]] && (echo "The downloaded recovery in image.bin doesn't look right." && kill $$)
sudo umount /dev/mapper/$rootpart || true
sudo mount -o ro -t ext4 /dev/mapper/$rootpart $tmpdir
chromeos_arch=$(file $tmpdir/bin/bash | awk '{print $7}' | sed 's/,//g')
echo "chromeos_arch is $chromeos_arch"
if [[ "$chromeos_arch" == "x86-64" ]]; then
ARCH=x86_64
MARCH=x86-64
LINUX32=
ARCH_LIB=lib64
CREW_PREFIX=/usr/local
CREW_LIB_PREFIX=$CREW_PREFIX/$ARCH_LIB
ALPINE_ARCH=amd64
PLATFORM="linux/amd64"
DOCKER_PLATFORM=amd64
elif [[ "$chromeos_arch" == "ARM" ]]; then
ARCH=armv7l
MARCH=armv7-a
LINUX32=
SHELL_LINUX32=/usr/local/bin/linux32
ARCH_LIB=lib
CREW_PREFIX=/usr/local
CREW_LIB_PREFIX=$CREW_PREFIX/$ARCH_LIB
ALPINE_ARCH=
PLATFORM="linux/arm/v7"
DOCKER_PLATFORM=armv7
elif [[ "$chromeos_arch" == "Intel" ]]; then
ARCH=i686
MARCH=i686
LINUX32=
SHELL_LINUX32=/usr/local/bin/linux32
ARCH_LIB=lib
CREW_PREFIX=/usr/local
CREW_LIB_PREFIX=$CREW_PREFIX/$ARCH_LIB
ALPINE_ARCH=i386
PLATFORM="linux/386"
DOCKER_PLATFORM=386
fi
}
import_to_Docker () {
if ! docker image ls | grep ${userid}/crewbase-${name}-${ARCH}.m${milestone} ; then
cd $tmpdir && sudo tar -c . | docker import --platform ${PLATFORM} - ${userid}/crewbase-${name}-${ARCH}.m${milestone}
fi
cd ..
sudo umount $tmpdir
rmdir $tmpdir
#rm image.bin
}
setup_docker_multiarch () {
# Registers docker multiarch.
# See https://github.com/multiarch/qemu-user-static
#docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
#[[ "$(uname -m)" == "x86_64" ]] && docker run --privileged --rm docker/binfmt:a7996909642ee92942dcd6cff44b9b95f08dad64
echo -e "{\n\"experimental\": \"enabled\"\n}" > ~/.docker/config.json
# Update buildx
echo "Updating buildx"
if [[ ! -f ~/.docker/cli-plugins/docker-buildx ]]; then
echo "~/.docker/cli-plugins/docker-buildx does not exist."
case "$(uname -m)" in
"aarch64")
BUILDXARCH=arm64
;;
"x86_64")
BUILDXARCH=amd64
;;
esac
sudo apt-get install -y jq
mkdir -p ~/.docker/cli-plugins
BUILDX_URL=$(curl https://api.github.com/repos/docker/buildx/releases/latest | jq -r '.assets[].browser_download_url' | grep ${BUILDXARCH} | grep linux)
curl -Lf $BUILDX_URL > ~/.docker/cli-plugins/docker-buildx
chmod +x ~/.docker/cli-plugins/docker-buildx
fi
docker buildx create --name ${ARCH}_builder --platform ${PLATFORM} --driver-opt network=host
docker buildx use ${ARCH}_builder
docker buildx inspect --bootstrap
# Registry is not configured in this script.
#if ! docker ps | grep registry; then
# Note that you will need to have
# {
# "insecure-registries": ["${REPOSITORY}"]
# }
# in /etc/docker/daemon.json for the registry to be accessible from buildx
#docker container stop registry #&& docker container rm -v registry
#docker run -d \
#--name registry \
#--restart always \
#-p 5000:5000 \
#registry:latest || true
#sleep 2 # Give this time to start up.
#fi
}
build_dockerfile () {
cd ${outdir}
cat <<EOF > ./${ARCH}/Dockerfile
# syntax=docker/dockerfile:experimental
ARG UID=1000
FROM ${REPOSITORY}/crewbase-${name}-${ARCH}.m${milestone}:${DOCKER_PLATFORM}
ENV LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$CREW_LIB_PREFIX
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
EOF
#[[ "$ARCH" == "i686" ]] && echo "RUN mkdir -p $CREW_PREFIX/bin" >> ./Dockerfile
#[[ "$ARCH" == "i686" ]] && echo "COPY curl_static_$ARCH $CREW_PREFIX/bin/curl_static" >> ./Dockerfile
#[[ "$ARCH" == "i686" ]] && echo "RUN cp /usr/bin/curl /usr/bin/curl_orig" >> ./Dockerfile
#[[ "$ARCH" == "i686" ]] && echo "COPY curl_static_$ARCH /usr/bin/curl" >> ./Dockerfile
cat <<EOF >> ./${ARCH}/Dockerfile
RUN --mount=type=bind,target=/input mkdir -p /usr/local/tmp && chown chronos /usr/local/tmp && \
ln -sf /input/built /usr/local/tmp/packages && \
mkdir -p ~chronos/.cache && chown chronos ~chronos/.cache && \
rm -f ~chronos/.cache/crewcache && \
passwd -d chronos && \
mkdir -p $CREW_PREFIX/sbin/ && mkdir -p $CREW_PREFIX/etc && ln -s /sbin/ldconfig $CREW_PREFIX/sbin/ && echo ${CREW_LIB_PREFIX} > $CREW_PREFIX/etc/ld.so.conf && chown -R chronos $CREW_PREFIX && \
echo "chronos ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers && \
echo -e 'export CREW_CACHE_OPT=1\nexport CFLAGS="-march=$MARCH"\n' >> ~chronos/.profile && chown chronos:chronos ~chronos/.profile && \
echo -e 'export LANG=en_US.UTF-8\nexport LANGUAGE=en_US.UTF-8\nexport LC_ALL=en_US.UTF-8\n' >> ~chronos/.bashrc && chown chronos:chronos ~chronos/.bashrc && \
echo -e 'export CFLAGS="-march=$MARCH"\n' >> ~chronos/.bashrc && chown chronos:chronos ~chronos/.bashrc && \
echo 'for i in \$(curl -Ls https://github.com/skycocker/chromebrew/raw/master/tools/core_packages.txt ) ; do ( yes | crew upgrade \$i || true ) ; done ; yes | crew reinstall python3 || true' >> ~chronos/install_core
SHELL ["sudo", "-n", "-u", "chronos", "/bin/bash", "-o", "pipefail", "-c"]
RUN set +h && curl -Ls https://github.com/skycocker/chromebrew/raw/master/install.sh | sed 's/ARCH="\$(uname -m)"/ARCH="${ARCH:-\$(uname -m)}"/' | sed 's/p11kit libtasn1 gnutls curl git icu4c libedit/p11kit libtasn1 gnutls curl git icu4c libedit gdbm libpipeline python3 util_linux/' | sed '/crew update/ a for i in \$(curl -Ls https://github.com/skycocker/chromebrew/raw/master/tools/core_packages.txt ) ; do ( yes | crew upgrade \$i || true ) ; done ; (yes | crew reinstall python3 || true )' | ARCH=$ARCH /bin/bash || true
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN --mount=type=bind,target=/input cp /input/preinstall/*.rb /usr/local/lib/crew/packages/ && chown chronos /usr/local/lib/crew/packages/*.rb || true
SHELL ["sudo", "-n", "-u", "chronos", "/bin/bash", "-o", "pipefail", "-c"]
env LD_LIBRARY_PATH=$CREW_LIB_PREFIX
# glibc has ldconfig
RUN yes | crew install glibc || true
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN echo $CREW_LIB_PREFIX > $CREW_PREFIX/etc/ld.so.conf && $CREW_PREFIX/sbin/ldconfig $CREW_PREFIX/etc/ld.so.conf
SHELL ["sudo", "-n", "-u", "chronos", "/bin/bash", "-o", "pipefail", "-c"]
RUN sed -i "s%LD_LIBRARY_PATH%sudo $CREW_PREFIX/sbin/ldconfig $CREW_PREFIX/etc/ld.so.conf ; LD_PRELOAD=$CREW_LIB_PREFIX/libpython3.9.so.1.0 LD_LIBRARY_PATH%g" /usr/local/lib/crew/packages/python3.rb && \
(yes | crew install python3 || true )
RUN yes | crew install binutils gdbm buildessential less most psmisc coreutils util_linux || true
SHELL ["sudo", "-n", "-u", "chronos", "/bin/bash", "-o", "pipefail", "-c"]
RUN yes | crew install docbook_xml || true
RUN yes | crew install libyaml || true
RUN yes | crew remove libiconv || true
RUN gem update --silent -N --system && \
gem install -N rubocop
RUN yes | crew install psmisc coreutils || true
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN --mount=type=bind,target=/input cp /input/postinstall/*.rb /usr/local/lib/crew/packages/ && chown chronos /usr/local/lib/crew/packages/*.rb || true
SHELL ["sudo", "-n", "-u", "chronos", "/bin/bash", "-o", "pipefail", "-c"]
RUN sudo rm ~chronos/.cache/crewcache/* || true
RUN echo "cd /usr/local/lib/crew/packages" >> ~chronos/.bashrc && \
echo "if [[ -e $CREW_PREFIX/bin/ccache ]]; then export PATH=\"/output/:$CREW_LIB_PREFIX/ccache/bin:$CREW_PREFIX/bin:$CREW_PREFIX/sbin:\\\$PATH\"; else export PATH=\"/output/:$CREW_PREFIX/bin:$CREW_PREFIX/sbin:\\\$PATH\" ; fi" >> ~chronos/.bashrc && \
echo "crew update" >> ~chronos/.bashrc && \
echo "sudo rm -rf ~chronos/.cache/crewcache ; mkdir -p ~chronos/.cache/crewcache" >> ~chronos/.bashrc && \
echo "cp /output/built/*.rb /usr/local/lib/crew/packages/" >> ~chronos/.bashrc && \
echo "sudo rm -f /usr/local/tmp/packages && ln -sf /output/built /usr/local/tmp/packages" >> ~chronos/.bashrc && \
echo "yes | crew upgrade || true" >> ~chronos/.bashrc && \
echo "for i in \\\$(ls /output/\\\$(hostname)-build/*.rb 2>/dev/null); do /output/crewbi \\\$i ; done" >> ~chronos/.bashrc && \
echo "for i in \\\$(ls /output/${ARCH}-build/*.rb 2>/dev/null); do /output/crewbi \\\$i ; done" >> ~chronos/.bashrc && \
echo "for i in \\\$(ls /output/autobuild/*.rb 2>/dev/null) ; do /output/crewbi \\\$i ; done" >> ~chronos/.bashrc && \
echo -e "#create 10 minute automatic logout to close session.\nTMOUT=600\nexport TMOUT" >> ~chronos/.bashrc
RUN ( sudo LD_LIBRARY_PATH=$CREW_LIB_PREFIX $CREW_PREFIX/sbin/ldconfig $CREW_PREFIX/etc/ld.so.conf || true ) && \
( $SHELL_LINUX32 uname -a || true )
EOF
#cat ./Dockerfile
}
get_curl() {
#[[ -f "$(abspath ${outdir})/curl_static_$ARCH" ]] && return 0
curl -Lf https://github.com/dtschan/curl-static/raw/master/build.sh -o "$(abspath ${outdir})"/curl.sh
sed -i "s/build alpine/build $ALPINE_ARCH\/alpine/" "$(abspath ${outdir})"/curl.sh && chmod +x "$(abspath ${outdir})"/curl.sh
$LINUX32 "$(abspath ${outdir})/"curl.sh && mv "$(abspath ${outdir})/"curl "$(abspath ${outdir})/"curl_static_$ARCH
}
build_docker_image_with_local_registry () {
docker ps
echo "Tag & Push starting in ..." && countdown "00:00:05"
if ! docker pull ${REPOSITORY}/crewbase-${name}-${ARCH}.m${milestone} ; then
docker tag ${userid}/crewbase-${name}-${ARCH}.m${milestone} ${REPOSITORY}/crewbase-${name}-${ARCH}.m${milestone}:${DOCKER_PLATFORM}
docker push ${REPOSITORY}/crewbase-${name}-${ARCH}.m${milestone}:${DOCKER_PLATFORM}
fi
#sed -i "s%FROM ${userid}/crewbase-${name}-${ARCH}.m${milestone}%FROM ${REPOSITORY}/crewbase-${name}-${ARCH}.m${milestone}%" Dockerfile
}
build_docker_image () {
docker image ls
dangling_images=$(docker images --filter "dangling=true" -q --no-trunc)
[[ ! -z "$dangling_images" ]] && docker rmi -f $(docker images --filter "dangling=true" -q --no-trunc)
# buildx has not yet implemented squash.
buildx_cmd="env BUILDKIT_PROGRESS=plain DOCKER_BUILDKIT=1 docker buildx build --builder ${ARCH}_builder --platform ${PLATFORM} --no-cache --progress=plain --tag ${REPOSITORY}/crewbase-${name}.m${milestone}:${DOCKER_PLATFORM} --push ./${ARCH}/"
echo $buildx_cmd
$buildx_cmd || echo "Docker Build Error."
}
make_docker_image_script () {
dockercmd="docker run --rm --platform "${PLATFORM}" -v \$(pwd):/output -h \$(hostname)-${ARCH} -it ${REPOSITORY}/crewbase-${name}.m${milestone}:${DOCKER_PLATFORM} $SHELL_LINUX32 sudo -i -u chronos bash -i"
[[ -f $(abspath ${outdir})/crewbase-${name}-${ARCH}.m${milestone}.sh ]] && rm $(abspath ${outdir})/crewbase-${name}-${ARCH}.m${milestone}.sh
echo -e "#!/bin/bash\n$dockercmd" > $(abspath ${outdir})/crewbase-${name}-${ARCH}.m${milestone}.sh && chmod +x $(abspath ${outdir})/crewbase-${name}-${ARCH}.m${milestone}.sh
}
enter_docker_image () {
echo "Running \"$dockercmd\" from \"$(abspath ${outdir})/crewbase-${name}-${ARCH}.m${milestone}.sh\""
echo "Entering in..." && countdown "00:00:30"
exec "$(abspath ${outdir})/crewbase-${name}-${ARCH}.m${milestone}.sh"
}
main () {
get_arch
mkdir ${outdir}/{autobuild,built,packages,preinstall,postinstall,src_cache,tmp,${ARCH}} &> /dev/null
import_to_Docker
setup_docker_multiarch
build_dockerfile
build_docker_image_with_local_registry
build_docker_image
make_docker_image_script
[[ -z "$JUST_BUILD" ]] && enter_docker_image
}
main
Raw
crewbi
#!/bin/bash
ARCH="$(uname -m)"
case "${ARCH}" in
"armv7l"|"armv8l")
ARCH="armv7l"
;;
esac
#echo "Need libyaml for rubocop gem"
#yes | crew install libyaml
gem install -N rubocop --conservative
ln -sf /output/crewbi /usr/local/bin/
crewfile="${1}"
pkg="${crewfile}"
crewname=$(basename "$crewfile")
crewname="${crewname%.*}"
pkg="${crewname}"
crewpath=$(dirname "$crewfile")
outpath="/output/built"
CleanBuildVars () {
local var="${1}"
[[ -n ${pkgver} ]] && var="${var//\$\{pkgver\}/$pkgver}"
[[ -n ${pkgbase} ]] && var="${var//\$\{pkgbase\}/$pkgbase}"
[[ -n ${pkgname} ]] && var=$(echo "${var}" | sed '/cd \$pkgname/d' )
[[ -n ${pkgver} ]] && var="${var//\$pkgver/$pkgver}"
[[ -n ${pkgname} ]] && var="${var//\$pkgname/$archpkgname}"
if [[ -n ${_ver} ]]; then
var=$(echo "${var}" | sed 's/#{_ver}/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/#{@_ver}/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/#{@_ver_}/'"$_ver_"'/g')
var=$(echo "${var}" | sed 's/#{@_ver_prelastdot}/'"$_ver_prelastdot"'/g')
var=$(echo "${var}" | sed 's/_ver/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/@_ver/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/@_ver_/'"$_ver_"'/g')
var=$(echo "${var}" | sed 's/@_ver_prelastdot/'"$_ver_prelastdot"'/g')
fi
if [[ -n ${version} ]];then
var=$(echo "${var}" | sed 's/@version/'"$version"'/g')
var=$(echo "${var}" | sed 's/#{version}/'"$version"'/g')
var=$(echo "${var}" | sed 's/@_ver/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/@_ver_/'"$_ver_"'/g')
var=$(echo "${var}" | sed 's/@_ver_prelastdot/'"$_ver_prelastdot"'/g')
fi
var=$(echo "${var}" | sed 's/"\$pkgdir"/#{CREW_DEST_DIR}/g' )
var=$(echo "${var}" | sed 's/\$pkgdir/#{CREW_DEST_DIR}/g' )
var=$(echo "${var}" | sed 's/^ *//g')
var=$(echo "${var}" | sed 's/--prefix=.*/--prefix=#{CREW_PREFIX} --libdir=#{CREW_LIB_PREFIX}/g' )
var=${var//arch-meson/meson}
var=$(echo "${var}" | sed 's/DESTDIR="\${pkgdir}"/DESTDIR=#{CREW_DEST_DIR}/g' )
echo "$var"
}
CleanBuildVars_shasum () {
local var="${1}"
[[ -n ${pkgver} ]] && var="${var//\$\{pkgver\}/$pkgver}"
[[ -n ${pkgbase} ]] && var="${var//\$\{pkgbase\}/$pkgbase}"
[[ -n ${pkgname} ]] && var=$(echo "${var}" | sed '/cd \$pkgname/d' )
[[ -n ${pkgver} ]] && var="${var//\$pkgver/$pkgver}"
[[ -n ${pkgname} ]] && var="${var//\$pkgname/$archpkgname}"
if [[ -n ${_ver} ]]; then
var=$(echo "${var}" | sed 's/#{_ver}/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/#{@_ver}/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/_ver/'"$_ver"'/g')
var=$(echo "${var}" | sed 's/@_ver/'"$_ver"'/g')
fi
if [[ -n ${version} ]];then
var=$(echo "${var}" | sed 's/@version/'"$version"'/g')
var=$(echo "${var}" | sed 's/#{version}/'"$version"'/g')
var=$(echo "${var}" | sed 's/@_ver/'"$_ver"'/g')
fi
var=$(echo "${var}" | sed 's/"\$pkgdir"/#{CREW_DEST_DIR}/g' )
var=$(echo "${var}" | sed 's/\$pkgdir/#{CREW_DEST_DIR}/g' )
var=$(echo "${var}" | sed 's/^ *//g')
var=$(echo "${var}" | sed 's/--prefix=.*/--prefix=#{CREW_PREFIX} --libdir=#{CREW_LIB_PREFIX}/g' )
var=${var//arch-meson/meson}
var=$(echo "${var}" | sed 's/DESTDIR="\${pkgdir}"/DESTDIR=#{CREW_DEST_DIR}/g' )
echo "$var"
}
get_old () {
field=
if ! grep -q "^[[:space:]]+@?${1} =" "$crewpath/${crewname}.rb"; then
local field="$(grep -m 1 "^[[:space:]]+@?${1}" "$crewpath/${crewname}.rb" | awk '{print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
else
local field="$(grep -m 1 "^[[:space:]]+@?${1} '\|^[[:space:]]+@?${1} \"" "$crewpath/${crewname}.rb")"
field=$(echo ${field#*=} | tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')
fi
CleanBuildVars "${field}"
}
get () {
field=
sterm="${1}"
# Check for equal sign in line
#if [[ $(cat "$crewpath/${crewname}.rb") =~ (Status:[[:space:]]200) ]];
if [[ $(grep -cE "^[[:space:]]+@?${sterm}[[:space:]]=" "$crewpath/${crewname}.rb") == "0" ]] ; then
noequalsign=1
local field="$(grep -E -m 1 "^[[:space:]]+@?${1}" "$crewpath/${crewname}.rb" | awk '{print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
else
#elif grep -q "^[[:space:]]+@?${1} =" "$crewpath/${crewname}.rb"; then
equalsign=1
local field="$(grep -E -m 1 "^[[:space:]]+@?${1} =" "$crewpath/${crewname}.rb" | awk 'BEGIN { FS = "=" } ; {print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
#else
#error_alt=1
#local field="$(grep -m 1 "^[[:space:]]+@?${1} '\|^[[:space:]]+@?${1} \"" "$crewpath/${crewname}.rb")"
#field=$(echo ${field#*=} | tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')
fi
CleanBuildVars "${field}"
}
get_shasum () {
field=
if ! grep -Eq "^[[:space:]]+@?${1}[[:space:]]=" "$crewpath/${crewname}.rb"; then
local field="$(grep -E -m 1 "^[[:space:]]+@?${1}" "$crewpath/${crewname}.rb" | awk '{print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
elif grep -Eq "^[[:space:]]+@?${1}[[:space:]]=" "$crewpath/${crewname}.rb"; then
local field="$(grep -E -m 1 "^[[:space:]]+@?${1}" "$crewpath/${crewname}.rb" | awk 'BEGIN { FS = "=" } ; {print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
else
local field="$(grep -E -m 1 "^[[:space:]]+@?${1} '\|^[[:space:]]+@?${1} \"" "$crewpath/${crewname}.rb")"
field=$(echo ${field#*=} | tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')
fi
CleanBuildVars "${field}"
}
get_shasum_new () {
field=
sterm="${1}"
# Check for equal sign in line
#if [[ $(cat "$crewpath/${crewname}.rb") =~ (Status:[[:space:]]200) ]];
if [[ $(grep -E -c "^[[:space:]]+@?${sterm}[[:space:]]=" "$crewpath/${crewname}.rb") == "0" ]] ; then
noequalsign=1
local field="$(grep -E -m 1 "^[[:space:]]+@?${1}" "$crewpath/${crewname}.rb" | awk '{print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
else
#elif grep -q "^[[:space:]]\+${1} =" "$crewpath/${crewname}.rb"; then
equalsign=1
local field="$(grep -E -m 1 "^[[:space:]]+@?${1} =" "$crewpath/${crewname}.rb" | grep -Eo '(http|https)://[^ >]+' | tr -d \' | tr -d \")"
#local field="$(grep -m 1 "^[[:space:]]\+${1} =" "$crewpath/${crewname}.rb" | awk 'BEGIN { FS = "=" } ; {print $2}'| tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')"
#else
#error_alt=1
#local field="$(grep -m 1 "^[[:space:]]\+${1} '\|^[[:space:]]\+${1} \"" "$crewpath/${crewname}.rb")"
#field=$(echo ${field#*=} | tr -d \' | tr -d \( | tr -d \) | tr -d \"| sed 's/{,.sig}//')
fi
CleanBuildVars "${field}"
}
cache_pkg_src() {
for i in source
do
source_hash="$(get ${i}_sha256)"
url="$(get ${i}_url)"
url=$(echo $url | sed "s #{_ver} $_ver g")
url=$(echo $url | sed "s #{version} $pkgver g")
[[ -n "$url" ]] && echo "url ${url}"
local source_filename=$(basename "${url}")
if [[ -e "/output/src_cache/${source_filename}" ]]; then
local_sha256hash=$(sha256sum "/output/src_cache/${source_filename}" | awk '{print $1}')
if [[ "${source_hash}" == "${local_sha256hash}" ]]; then
ln -sf "/output/src_cache/${source_filename}" ~/.cache/crewcache/
return 0
fi
fi
echo "Caching source package"
local outfile=$(cd /tmp && /usr/bin/curl --retry 3 -OLf "$url" -w %{filename_effective})
dlsha256hash=$(sha256sum "/tmp/$outfile" | awk '{print $1}')
if [[ "${source_hash}" == "${dlsha256hash}" ]]; then
cp "/tmp/${outfile}" "/output/src_cache/${source_filename}"
rm "/usr/local/tmp/crew/${source_filename}" && ln -sf "/tmp/${outfile}" /usr/local/tmp/crew/
else
delim=" "
newhash="${i}_sha256${delim}'$dlsha256hash'"
echo "$newhash"
if grep -q "${i}_sha256" "$crewpath/${crewname}.rb"; then
sed -i 's/^\([[:blank:]]*\)'"$i"'_sha256.*/\1'"$newhash"'/' "$crewpath/${crewname}.rb"
else
sed -i 's/^\([[:blank:]]*\)'"$i"'_url.*/a\1'"$newhash"'/' "$crewpath/${crewname}.rb"
fi
cp "/tmp/${outfile}" "/output/src_cache/${source_filename}"
rm "/usr/local/tmp/crew/${source_filename}" && ln -sf "/tmp/${outfile}" /usr/local/tmp/crew/
add_other_hashes_to_file
fi
done
}
GetSHA256 () {
LD_LIBRARY_PATH=/usr/lib64 /usr/bin/curl -Lsf "${1}" | sha256sum -b - | awk '{print $1}'
}
add_other_hashes_to_file() {
for i in patch
do
echo $i
url="$(get_shasum ${i}_url)"
url=$(echo $url | sed "s #{_ver} $_ver g")
[[ -n "$url" ]] && echo "url ${url}"
sha256=$(GetSHA256 "$url")
if [[ "$i" == "source" ]]; then
delim=" "
else
delim=" = "
fi
newhash="${i}_sha256${delim}'$sha256'"
echo "$newhash"
if grep -q "${i}_sha256" "$crewpath/${crewname}.rb"; then
sed -i 's/^\([[:blank:]]*\)'"$i"'_sha256.*/\1'"$newhash"'/' "$crewpath/${crewname}.rb"
else
sed -i 's/^\([[:blank:]]*\)'"$i"'_url.*/a\1'"$newhash"'/' "$crewpath/${crewname}.rb"
fi
url=
sha256=
newhash=
done
for i in $(grep "^ \+url_*" "$crewpath/${crewname}.rb" | awk '{print $1}' | sed 's/url_//g');
do
url=$(get_shasum_new url_"$i")
[[ -n "$url" ]] && echo "url $url"
sha256=$(GetSHA256 "$url")
delim=" = "
newhash="sha256sum_${i}${delim}'$sha256'"
if [[ $sha256 != "-1" ]]; then
echo "$newhash"
if grep -q "sha256sum_${i}" "$crewpath/${crewname}.rb"; then
sed -i 's/^\([[:blank:]]*\)sha256sum_'"$i"'\b.*/\1'"$newhash"'/' "$crewpath/${crewname}.rb"
else
sed -i 's/^\([[:blank:]]*\)url_'"$i"'\b.*/a\1'"$newhash"'/' "$crewpath/${crewname}.rb"
fi
fi
url=
sha256=
newhash=
done
}
main () {
echo "pkg is $pkg"
if [[ -f "$crewpath/${crewname}.rb" ]] ; then
if [[ "$crewpath" == "." ]] && [[ "$(pwd)" == "/usr/local/lib/crew/packages" ]]; then
crewpath="/usr/local/lib/crew/packages"
fi
else
echo "missing package file for ${pkg}." && exit 1
fi
mkdir -p /output/{done,work}-$ARCH/
if { [[ "$crewpath" == "/output/autobuild" ]] || [[ "$crewpath" == "$outpath" ]] ; } ; then
[[ -e "/output/done-$ARCH/${crewname}.rb" ]] && echo "already done" && exit 0
[[ -e "/output/work-$ARCH/${crewname}.rb" ]] && echo "working" && exit 0
cp "$crewpath/${crewname}.rb" "/output/work-$ARCH/${crewname}.rb"
fi
if [[ "$crewpath" != "/usr/local/lib/crew/packages" ]]; then
cp "$crewpath/${crewname}.rb" /usr/local/lib/crew/packages/
ogcrewpath="$crewpath"
crewpath="/usr/local/lib/crew/packages"
fi
if ! [[ -f "$crewpath/${crewname}.rb" ]] ; then
exit 1
crewpath="/usr/local/lib/crew/packages"
fi
_ver="$(get _ver)"
_ver_="${_ver//./_}"
_ver_prelastdot="${_ver%.*}"
pkgver="$(get version)"
pkgver="${pkgver//@}"
echo "pkgver is ${pkgver}"
mkdir -p /output/{build,built,logs,packages,src_cache,tmp}
tmpdir=$(mktemp -d docker_XXXX -p "/output/tmp")
cp ${crewpath}/${pkg}.rb "$tmpdir"/
# sanitize package
sed -i "/^[[:blank:]]*compatibility/d" "$tmpdir"/${pkg}.rb
sed -i "/^[[:blank:]]*version/ a compatibility 'all'" "$tmpdir"/${pkg}.rb
cat <<'EOF'> "$tmpdir"/.rubocop.yml
# The behavior of RuboCop can be controlled via the .rubocop.yml
# configuration file. It makes it possible to enable/disable
# certain cops (checks) and to alter their behavior if they accept
# any parameters. The file can be placed either in your home
# directory or in some project directory.
#
# RuboCop will start looking for the configuration file in the directory
# where the inspected file is and continue its way up to the root directory.
#
# See https://docs.rubocop.org/rubocop/configuration
AllCops:
NewCops: enable
Layout/HashAlignment:
EnforcedHashRocketStyle:
- separator
EnforcedColonStyle:
- separator
Layout/FirstHashElementIndentation:
EnforcedStyle: consistent
Style/FrozenStringLiteralComment:
Enabled: false
Style/Documentation:
Enabled: false
Naming/VariableNumber:
Enabled: false
Naming/ClassAndModuleCamelCase:
Enabled: false
Metrics/MethodLength:
Enabled: false
Layout/LineLength:
IgnoredPatterns: ['description']
EOF
rubocop -c "$tmpdir"/.rubocop.yml -A "$tmpdir"/${pkg}.rb
cache_pkg_src
#mkdir -p /output/{packages,built} && cd /output/packages
cd "$tmpdir"
if [ -L ~/.cache/crewcache ]; then
if readlink -q ~/.cache/crewcache >/dev/null ; then
sudo rm ~/.cache/crewcache && sudo mkdir -p ~/.cache/crewcache
sudo chown -R chronos ~/.cache/crewcache
fi
fi
#rm -f ~/.cache/crewcache && ln -sf $tmpdir ~/.cache/crewcache
for i in $(ls /output/src_cache/*.xz 2>/dev/null ; ls /output/src_cache/*.zip 2>/dev/null ; ls /output/src_cache/*.gz 2>/dev/null); do ln -sf $i ~/.cache/crewcache/ ; done
for i in $(ls /output/packages/*.xz 2>/dev/null ; ls /output/packages/*.zip 2>/dev/null ; ls /output/packages/*.gz 2>/dev/null) ; do ln -sf $i ~/.cache/crewcache/ ; done
for i in $(ls /output/built/*.xz 2>/dev/null) ; do ln -sf $i ~/.cache/crewcache/ ; done
#crew remove ${pkg}
rm -rf /usr/local/tmp/crew/*.dir/
#rm -f /output/packages/$pkg*.tar.xz
echo "Compiling. output saved to logs/${pkg}-${pkgver}-${ARCH}.log in output dir."
cp "$tmpdir"/${pkg}.rb /usr/local/lib/crew/packages/
yes | nice -n 18 crew build -k ${pkg} | tee /output/logs/${pkg}-${pkgver}-${ARCH}.log
if ls ${pkg}-"${pkgver}"-chromeos-${ARCH}.tar.xz ; then
# Manual install
#crew remove ${pkg} && tar fx ${pkg}-${pkgver}-chromeos-${ARCH}.tar.xz -C /
#crew install -s ${pkg}
cp "$tmpdir"/$(ls ${pkg}-"${pkgver}"-chromeos-${ARCH}.tar.xz) ${outpath}/
#cp "$tmpdir"/$(ls ${pkg}*${pkgver}*"$ARCH".tar.xz.sha256) ${outpath}/
# Generate binary_url section of package file
aarch64_filename="$(cd ${outpath}/ && ls ${pkg}-"${pkgver}"-chromeos-*.tar.xz | grep aarch64 | grep ${pkgver})"
if [[ ! -z "$aarch64_filename" ]]; then
aarch64_file="file:///usr/local/tmp/packages/${aarch64_filename}"
aarch64_hash="$(sha256sum ${outpath}/${aarch64_filename}| awk '{print $1}')"
else
aarch64_file=
aarch64_hash=
fi
armv7l_filename="$(cd ${outpath}/ && ls ${pkg}-"${pkgver}"-chromeos-*.tar.xz | grep armv7l | grep ${pkgver})"
if [[ ! -z "$armv7l_filename" ]]; then
armv7l_file="file:///usr/local/tmp/packages/${armv7l_filename}"
armv7l_hash="$(sha256sum ${outpath}/${armv7l_filename}| awk '{print $1}')"
else
armv7l_file=
armv7l_hash=
fi
i686_filename="$(cd ${outpath}/ && ls ${pkg}-"${pkgver}"-chromeos-*.tar.xz | grep i686 | grep ${pkgver})"
if [[ ! -z "$i686_filename" ]]; then
i686_file="file:///usr/local/tmp/packages/${i686_filename}"
i686_hash="$(sha256sum ${outpath}/${i686_filename}| awk '{print $1}')"
else
i686_file=
i686_hash=
fi
x86_64_filename="$(cd ${outpath}/ && ls ${pkg}-"${pkgver}"-chromeos-*.tar.xz | grep x86_64 | grep ${pkgver})"
if [[ ! -z "$x86_64_filename" ]]; then
x86_64_file="file:///usr/local/tmp/packages/${x86_64_filename}"
x86_64_hash="$(sha256sum ${outpath}/${x86_64_filename}| awk '{print $1}')"
else
x86_64_file=
x86_64_hash=
fi
echo -e "\n binary_url ({" > ${pkg}-${pkgver}.files
[[ ! -z "$armv7l_file" ]] && echo -e " aarch64: '$armv7l_file'," >> ${pkg}-${pkgver}.files
[[ ! -z "$armv7l_file" ]] && echo -e " armv7l: '$armv7l_file'," >> ${pkg}-${pkgver}.files
[[ ! -z "$i686_file" ]] && echo -e " i686: '$i686_file'," >> ${pkg}-${pkgver}.files
[[ ! -z "$x86_64_file" ]] && echo -e " x86_64: '$x86_64_file'," >> ${pkg}-${pkgver}.files
echo -e " })" >> ${pkg}-${pkgver}.files
echo -e " binary_sha256 ({" >> ${pkg}-${pkgver}.files
[[ ! -z "$armv7l_hash" ]] && echo -e " aarch64: '$armv7l_hash'," >> ${pkg}-${pkgver}.files
[[ ! -z "$armv7l_hash" ]] && echo -e " armv7l: '$armv7l_hash'," >> ${pkg}-${pkgver}.files
[[ ! -z "$i686_hash" ]] && echo -e " i686: '$i686_hash'," >> ${pkg}-${pkgver}.files
[[ ! -z "$x86_64_hash" ]] && echo -e " x86_64: '$x86_64_hash'," >> ${pkg}-${pkgver}.files
echo -e " })" >> ${pkg}-${pkgver}.files
# For now just clean out binary_url section.
sed -i "/({/,/})/d" "$tmpdir"/${pkg}.rb
if ! grep -q binary_url "$tmpdir"/${pkg}.rb; then
sed -i -e "/source_sha256/r ${pkg}-${pkgver}.files" "$tmpdir"/${pkg}.rb
elif [[ ! -z "$aarch64_hash" ]]; then
aarch64_oldhash=$(grep -m 3 aarch64 "$tmpdir"/${pkg}.rb | tail -n 1 | awk '{print $2}' | tr -d \' | sed 's/,//g')
[[ ! -z "$aarch64_oldhash" ]] && sed -i "s#$aarch64_oldhash#$armv7l_hash#" "$tmpdir"/${pkg}.rb
elif [[ ! -z "$armv7l_hash" ]]; then
armv7l_oldhash=$(grep -m 3 armv7l "$tmpdir"/${pkg}.rb | tail -n 1 | awk '{print $2}' | tr -d \' | sed 's/,//g')
[[ ! -z "$armv7l_oldhash" ]] && sed -i "s#$armv7l_oldhash#$armv7l_hash#" "$tmpdir"/${pkg}.rb
elif [[ ! -z "$i686_hash" ]]; then
i686_oldhash=$(grep -m 3 i686 "$tmpdir"/${pkg}.rb | tail -n 1 | awk '{print $2}' | tr -d \' | sed 's/,//g')
[[ ! -z "$i686_oldhash" ]] && sed -i "s#$i686_oldhash#$i686_hash#" "$tmpdir"/${pkg}.rb
elif [[ ! -z "$x86_64_hash" ]]; then
x86_64_oldhash=$(grep -m 3 x86_64 "$tmpdir"/${pkg}.rb | tail -n 1 | awk '{print $2}' | tr -d \' | sed 's/,//g')
[[ ! -z "$x86_64_oldhash" ]] && sed -i "s#$x86_64_oldhash#$x86_64_hash#" "$tmpdir"/${pkg}.rb
fi
#ls -aFl "$tmpdir"
cp -f "$tmpdir"/${pkg}.rb ${outpath}/
cp "$tmpdir"/$(ls ${pkg}-"${pkgver}"-chromeos-${ARCH}.tar.xz) ${outpath}/
cp "$tmpdir"/$(ls ${pkg}-"${pkgver}"-chromeos-${ARCH}.tar.xz) ${outpath}/../release/$ARCH/
for i in $(ls ${outpath}/${pkg}-"${pkgver}"-chromeos-${ARCH}.tar.xz 2>/dev/null)
do
ln -sf $i ~/.cache/crewcache/
done
cp ${outpath}/${pkg}.rb /usr/local/lib/crew/packages/
rm /usr/local/tmp/packages && ln -sf ${outpath} /usr/local/tmp/packages
cp "$crewpath/${crewname}.rb" "/output/done-$ARCH/${crewname}.rb" && rm "/output/work-$ARCH/${crewname}.rb"
#cp $outpath/
#if { [[ ! -z "$armv7l_hash" ]] && [[ ! -z "$i686_hash" ]] && [[ ! -z "$x86_64_hash" ]] ; }; then
#if { [[ ! -z "$armv7l_hash" ]] && [[ ! -z "$x86_64_hash" ]] ; }; then
if { [[ ! -z "$x86_64_hash" ]] ; }; then
for archvar in x86_64 i686 armv7l ;
do
if [[ -e "$outpath"/${pkg}-"${pkgver}"-chromeos-${archvar}.tar.xz ]]; then
cp "$outpath"/${pkg}-"${pkgver}"-chromeos-${archvar}.tar.xz ${outpath}/../release/$archvar/
sed -i "s,file:///usr/local/tmp/packages/,https://dl.bintray.com/chromebrew/chromebrew/,g" "$tmpdir"/${pkg}.rb
#rubocop -c "$tmpdir"/.rubocop.yml -A "$tmpdir"/${pkg}.rb
#cp "$tmpdir"/${pkg}.rb ${outpath}/../forupload/
fi
done
for i in $(curl -Lsf https://github.com/skycocker/chromebrew/raw/master/tools/core_packages.txt)
do
sed -i "/^[[:blank:]]*depends_on '$i'/d" "$tmpdir"/${pkg}.rb
sed -i "/^[[:blank:]]*depends_on \"$i\"/d" "$tmpdir"/${pkg}.rb
done
sed -i "/^[[:blank:]]*compatibility/d" "$tmpdir"/${pkg}.rb
if { [[ ! -z "$armv7l_hash" ]] && [[ ! -z "$i686_hash" ]] && [[ ! -z "$x86_64_hash" ]] ; }; then
sed -i "/^[[:blank:]]*version/ a compatibility 'all'" "$tmpdir"/${pkg}.rb
elif { [[ ! -z "$armv7l_hash" ]] && [[ ! -z "$x86_64_hash" ]] ; }; then
sed -i "/^[[:blank:]]*version/ a compatibility 'x86_64 aarch64 armv7l'" "$tmpdir"/${pkg}.rb
elif { [[ ! -z "$x86_64_hash" ]] ; }; then
sed -i "/^[[:blank:]]*version/ a compatibility 'x86_64'" "$tmpdir"/${pkg}.rb
fi
rubocop -c "$tmpdir"/.rubocop.yml -A "$tmpdir"/${pkg}.rb
cp "$tmpdir"/${pkg}.rb ${outpath}/../forupload/
if { [[ ! -z "$armv7l_hash" ]] && [[ ! -z "$i686_hash" ]] && [[ ! -z "$x86_64_hash" ]] ; }; then
rm "/output/autobuild/${crewname}.rb" &>/dev/null
fi
fi
if ! crew install ${pkg} 2>&1 | grep -vzq "already installed, skipping..." ; then
crew reinstall ${pkg}
fi
else
echo "There was an error in generating the package!"
rm "/output/work-$ARCH/${crewname}.rb"
tail /output/logs/${pkg}-${pkgver}-${ARCH}.log
fi
cd /usr/local/lib/crew/packages/
#echo "tmpdir is $tmpdir"
rm -rf "$tmpdir"
}
main
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment