365 lines
11 KiB
Bash
365 lines
11 KiB
Bash
#!/usr/bin/env zsh
|
|
#
|
|
# This script does not have a stable API.
|
|
#
|
|
# Usage: mbuild [-b git-ref] [kernel-arch]...
|
|
#
|
|
# Builds a bunch of gitstatusd-* binaries. Without arguments builds binaries
|
|
# for all platforms. git-ref defaults to src.
|
|
#
|
|
# Before using this script you need to set up build servers and list them
|
|
# in ~/.ssh/config. There should be a Host entry for every value of `assets`
|
|
# association defined below. VMs and cloud instances work as well as physical
|
|
# machines, including localhost. As long as the machine has been set up as
|
|
# described below and you can SSH to it without password, it should work.
|
|
#
|
|
# ===[ Build Server Setup ]===
|
|
#
|
|
# Linux
|
|
#
|
|
# - Install docker.
|
|
# $ apt install docker.io # adjust appropriately if there is no `apt`
|
|
# $ usermod -aG docker $USER # not needed if going to build as root
|
|
# - Install git.
|
|
# $ apt install git # adjust appropriately if there is no `apt`
|
|
#
|
|
# macOS
|
|
#
|
|
# - Install compiler tools:
|
|
# $ xcode-select --install
|
|
# - Install homebrew: https://brew.sh/.
|
|
# $ bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
|
|
#
|
|
# FreeBSD
|
|
#
|
|
# - Install git.
|
|
# $ pkg install git
|
|
#
|
|
# Windows
|
|
#
|
|
# - Disable Windows Defender (optional).
|
|
# ps> Set-MpPreference -DisableRealtimeMonitoring $true
|
|
# - Install 64-bit and 32-bit msys2: https://www.msys2.org/wiki/MSYS2-installation/.
|
|
# - Open each of them after installation, type `pacman -Syu --noconfirm` and close the window.
|
|
# - Then run in powershell while having no msys2 or cygwin windows open:
|
|
# ps> C:\msys32\autorebase.bat
|
|
# ps> C:\msys64\autorebase.bat
|
|
# - Install 64-bit and 32-bit cygwin: https://cygwin.com/install.html.
|
|
# - Choose to install 32-bit to c:/cygwin32 instead of the default c:/cygwin.
|
|
# - Select these packages: binutils, cmake, gcc-core, gcc-g++, git, make, perl, wget.
|
|
#
|
|
# IMPORTANT: Install msys2 and cygwin one at a time.
|
|
#
|
|
# IMPORTANT: msys2 builder can reboot the build machine.
|
|
#
|
|
# Option 1: OpenSSH for Windows
|
|
#
|
|
# - Install OpenSSH: https://docs.microsoft.com/en-us/windows-server/administration/openssh/openssh_install_firstuse.
|
|
# ps> Add-WindowsCapability -Online -Name OpenSSH.Server~~~~0.0.1.0
|
|
# ps> Start-Service sshd
|
|
# ps> Set-Service -Name sshd -StartupType 'Automatic'
|
|
# - Enable publickey authentication: https://stackoverflow.com/a/50502015/1095235.
|
|
# ps> cd $env:USERPROFILE
|
|
# ps> mkdir .ssh
|
|
# ps> notepad.exe .ssh/authorized_keys
|
|
# - Paste your public key, save, close.
|
|
# ps> icacls .ssh/authorized_keys /inheritance:r
|
|
# ps> notepad.exe C:\ProgramData\ssh\sshd_config
|
|
# - Comment out these two lines, save, close:
|
|
# # Match Group administrators
|
|
# # AuthorizedKeysFile __PROGRAMDATA__/ssh/administrators_authorized_keys
|
|
# ps> Restart-Service sshd
|
|
#
|
|
# Option 2: OpenSSH from WSL
|
|
#
|
|
# - Install WSL.
|
|
# - Install Ubuntu.
|
|
# - Install sshd.
|
|
# $ apt install openssh-server
|
|
# $ dpkg-reconfigure openssh-server
|
|
# $ cat >/etc/ssh/sshd_config <<\END
|
|
# ClientAliveInterval 60
|
|
# AcceptEnv TERM LANG LC_*
|
|
# PermitRootLogin no
|
|
# AllowTcpForwarding no
|
|
# AllowAgentForwarding no
|
|
# AllowStreamLocalForwarding no
|
|
# AuthenticationMethods publickey
|
|
# END
|
|
# service ssh --full-restart
|
|
# - Add your public ssh key to ~/.ssh/authorized_keys.
|
|
# - Make `sshd` start when Windows boots.
|
|
|
|
'emulate' '-L' 'zsh' '-o' 'no_aliases' '-o' 'err_return'
|
|
setopt no_unset extended_glob pipe_fail prompt_percent typeset_silent \
|
|
no_prompt_subst no_prompt_bang pushd_silent warn_create_global
|
|
|
|
autoload -Uz is-at-least
|
|
|
|
if ! is-at-least 5.1 || [[ $ZSH_VERSION == 5.4.* ]]; then
|
|
print -ru2 -- "[error] unsupported zsh version: $ZSH_VERSION"
|
|
return 1
|
|
fi
|
|
|
|
zmodload zsh/system
|
|
|
|
local -r git_url='https://github.com/romkatv/gitstatus.git'
|
|
|
|
local -rA assets=(
|
|
# target kernel-arch hostname of the build machine
|
|
cygwin_nt-10.0-i686 build-windows-x86_64
|
|
cygwin_nt-10.0-x86_64 build-windows-x86_64
|
|
msys_nt-10.0-i686 build-windows-x86_64
|
|
msys_nt-10.0-x86_64 build-windows-x86_64
|
|
darwin-x86_64 build-macos-x86_64
|
|
freebsd-amd64 build-freebsd-amd64
|
|
linux-aarch64 build-linux-aarch64
|
|
linux-armv6l build-linux-armv7l
|
|
linux-armv7l build-linux-armv7l
|
|
linux-i686 build-linux-x86_64
|
|
linux-ppc64le build-linux-ppc64le
|
|
linux-x86_64 build-linux-x86_64
|
|
)
|
|
|
|
local -rA protocol=(
|
|
'cygwin_nt-10.0-*' windows
|
|
'msys_nt-10.0-*' windows
|
|
'darwin-*' unix
|
|
'freebsd-*' unix
|
|
'linux-*' unix
|
|
)
|
|
|
|
local -r rootdir=${ZSH_SCRIPT:h}
|
|
local -r logs=$rootdir/logs
|
|
local -r locks=$rootdir/locks
|
|
local -r binaries=$rootdir/usrbin
|
|
|
|
function usage() {
|
|
print -r -- 'usage: mbuild [-b REF] [KERNEL-ARCH]...'
|
|
}
|
|
|
|
local OPTARG opt git_ref=src
|
|
local -i OPTIND
|
|
while getopts ":b:h" opt; do
|
|
case $opt in
|
|
h) usage; return 0;;
|
|
b) [[ -n $OPTARG ]]; git_ref=$OPTARG;;
|
|
\?) print -ru2 -- "mbuild: invalid option: -$OPTARG" ; return 1;;
|
|
:) print -ru2 -- "mbuild: missing required argument: -$OPTARG"; return 1;;
|
|
*) print -ru2 -- "mbuild: invalid option: -$opt" ; return 1;;
|
|
esac
|
|
done
|
|
|
|
shift $((OPTIND - 1))
|
|
|
|
(( $# )) || set -- ${(ko)assets}
|
|
set -- ${(u)@}
|
|
|
|
local platform
|
|
for platform; do
|
|
if (( ! $+assets[$platform] )); then
|
|
print -ru2 -- "mbuild: invalid platform: $platform"
|
|
return 1
|
|
fi
|
|
done
|
|
|
|
local build='
|
|
rm -rf gitstatus
|
|
git clone --recursive --shallow-submodules --depth=1 -b '$git_ref' '$git_url'
|
|
cd gitstatus
|
|
if command -v zsh >/dev/null 2>&1; then
|
|
sh=zsh
|
|
elif command -v dash >/dev/null 2>&1; then
|
|
sh=dash
|
|
elif command -v ash >/dev/null 2>&1; then
|
|
sh=ash
|
|
else
|
|
sh=sh
|
|
fi
|
|
$sh -x ./build -m '
|
|
|
|
function build-unix() {
|
|
local intro flags=(-sw)
|
|
case $2 in
|
|
linux-ppc64le) ;;
|
|
linux-*) flags+=(-d docker);;
|
|
darwin-arm64) intro='PATH="/opt/homebrew/bin:$PATH"';;
|
|
darwin-*) intro='PATH="/usr/local/bin:$PATH"';;
|
|
esac
|
|
ssh $1 -- /bin/sh -uex <<<"
|
|
$intro
|
|
cd /tmp
|
|
$build ${2##*-} ${(j: :)${(@q)flags}}"
|
|
scp $1:/tmp/gitstatus/usrbin/gitstatusd $binaries/gitstatusd-$2
|
|
}
|
|
|
|
function build-windows() {
|
|
local shell=$(ssh $1 'echo $0')
|
|
if [[ $shell == '$0'* ]]; then
|
|
local c='c:'
|
|
else
|
|
local c='/mnt/c'
|
|
fi
|
|
|
|
local tmp env bin intro flags=(-w)
|
|
case $2 in
|
|
cygwin_nt-10.0-i686) bin='cygwin32/bin' ;|
|
|
cygwin_nt-10.0-x86_64) bin='cygwin64/bin' ;|
|
|
msys_nt-10.0-i686) bin='msys32/usr/bin';|
|
|
msys_nt-10.0-x86_64) bin='msys64/usr/bin';|
|
|
cygwin_nt-10.0-*)
|
|
tmp='/cygdrive/c/tmp'
|
|
;|
|
|
msys_nt-10.0-*)
|
|
flags+=(-s)
|
|
tmp='/c/tmp'
|
|
env='MSYSTEM=MSYS'
|
|
intro='pacman -Syu --noconfirm; pacman -S --needed --noconfirm git; '
|
|
intro+='PATH="$PATH:/usr/bin/site_perl:/usr/bin/vendor_perl:/usr/bin/core_perl"'
|
|
while true; do
|
|
# TODO: run autorebase only when getting an error that can be fixed by autorebasing.
|
|
break
|
|
local out
|
|
out="$(ssh $1 cmd.exe "$c/${bin%%/*}/autorebase.bat" 2>&1)"
|
|
[[ $out == *"The following DLLs couldn't be rebased"* ]] || break
|
|
# Reboot to get rid of whatever is using those DLLs.
|
|
ssh $1 powershell.exe <<<'Restart-Computer -Force' || true
|
|
sleep 30
|
|
while ! ssh $1 <<<''; do sleep 5; done
|
|
done
|
|
() {
|
|
while true; do
|
|
local -i fd
|
|
exec {fd}< <(
|
|
ssh $1 $c/$bin/env.exe $env c:/$bin/bash.exe -l 2>&1 <<<"
|
|
pacman -Syu --noconfirm
|
|
exit")
|
|
{
|
|
local line
|
|
while true; do
|
|
IFS= read -u $fd -r line || return 0
|
|
if [[ $line == *"warning: terminate MSYS2"* ]]; then
|
|
# At this point the machine is hosed. A rogue process with a corrupted name
|
|
# is eating all CPU. The top SSH connection won't terminate on its own.
|
|
ssh $1 powershell.exe <<<'Restart-Computer -Force' || true
|
|
sleep 30
|
|
while ! ssh $1 <<<''; do sleep 5; done
|
|
break
|
|
fi
|
|
done
|
|
} always {
|
|
exec {fd}<&-
|
|
kill -- -$sysparams[procsubstpid] 2>/dev/null || true
|
|
}
|
|
done
|
|
} "$@"
|
|
;|
|
|
esac
|
|
|
|
ssh $1 $c/$bin/env.exe $env c:/$bin/bash.exe -l <<<"
|
|
set -uex
|
|
$intro
|
|
mkdir -p -- $tmp
|
|
cd -- $tmp
|
|
$build ${2##*-} ${(j: :)${(@q)flags}}
|
|
exit"
|
|
scp $1:$c/tmp/gitstatus/usrbin/gitstatusd $binaries/gitstatusd-$2
|
|
chmod +x $binaries/gitstatusd-$2
|
|
}
|
|
|
|
function build() (
|
|
setopt xtrace
|
|
local platform=$1
|
|
local machine=$assets[$platform]
|
|
print -n >>$locks/$machine
|
|
zsystem flock $locks/$machine
|
|
build-${protocol[(k)$platform]} $machine $platform
|
|
local tmp=gitstatusd-$platform.tmp.$$.tar.gz
|
|
( cd -q -- $binaries; tar --owner=0 --group=0 -I 'gzip -9' -cf $tmp gitstatusd-$platform )
|
|
mv -f -- $binaries/$tmp $binaries/gitstatusd-$platform.tar.gz
|
|
)
|
|
|
|
function mbuild() {
|
|
local platform pid pids=()
|
|
for platform; do
|
|
build $platform &>$logs/$platform &
|
|
print -r -- "starting build for $platform on $assets[$platform] (pid $!)"
|
|
pids+=($platform $!)
|
|
done
|
|
local failed=()
|
|
for platform pid in $pids; do
|
|
print -rn -- "$platform => "
|
|
if wait $pid; then
|
|
print -r -- "ok"
|
|
else
|
|
print -r -- "error"
|
|
failed+=$platform
|
|
fi
|
|
done
|
|
(( $#failed )) || return 0
|
|
print
|
|
print -r -- "Error logs:"
|
|
print
|
|
for platform in $failed; do
|
|
print -r -- " $platform => $logs/$platform"
|
|
done
|
|
return 1
|
|
}
|
|
|
|
# Copied from https://github.com/romkatv/run-process-tree.
|
|
function run-process-tree() {
|
|
zmodload zsh/parameter zsh/param/private || return
|
|
local -P opt=(${(kv)options[@]}) || return
|
|
local -P pat=(${patchars[@]}) || return
|
|
local -P dis_pat=(${dis_patchars[@]}) || return
|
|
emulate -L zsh -o err_return || return
|
|
setopt monitor traps_async pipe_fail no_unset
|
|
zmodload zsh/system
|
|
|
|
if (( $# == 0 )); then
|
|
print -ru2 -- 'usage: run-process-tree command [arg]...'
|
|
return 1
|
|
fi
|
|
|
|
local -P stdout REPLY
|
|
exec {stdout}>&1
|
|
{
|
|
{
|
|
local -Pi pipe
|
|
local -P gid=$sysparams[pid]
|
|
local -P sig=(ABRT EXIT HUP ILL INT PIPE QUIT TERM ZERR)
|
|
local -P trap=(trap "trap - $sig; kill -- -$sysparams[pid]" $sig)
|
|
|
|
exec {pipe}>&1 1>&$stdout
|
|
$trap
|
|
|
|
{
|
|
$trap
|
|
while sleep 1 && print -u $pipe .; do; done
|
|
} 2>/dev/null &
|
|
local -Pi watchdog=$!
|
|
|
|
{
|
|
trap - ZERR
|
|
exec {pipe}>&-
|
|
enable -p -- $pat
|
|
disable -p -- $dis_pat
|
|
options=($opt zle off monitor off)
|
|
"$@"
|
|
} &
|
|
local -Pi ret
|
|
wait $! || ret=$?
|
|
|
|
trap "exit $ret" TERM
|
|
kill $watchdog
|
|
wait $watchdog
|
|
return ret
|
|
} | while read; do; done || return
|
|
} always {
|
|
exec {stdout}>&-
|
|
}
|
|
}
|
|
|
|
mkdir -p -- $logs $locks $binaries
|
|
run-process-tree mbuild $@
|