diff options
author | 2018-12-15 18:08:30 -0500 | |
---|---|---|
committer | 2018-12-15 18:08:30 -0500 | |
commit | 2e15a5491a1383b89820853f8acb25f44dccac53 (patch) | |
tree | 38c74ba6bedd39bf819cef9c950da40286fcf22d | |
parent | Add Robin's ha-proxy bits for non-cloud LB. (diff) | |
download | antarus-2e15a5491a1383b89820853f8acb25f44dccac53.tar.gz antarus-2e15a5491a1383b89820853f8acb25f44dccac53.tar.bz2 antarus-2e15a5491a1383b89820853f8acb25f44dccac53.zip |
Various rsync node improvements.
WAIT_TIME=30m to test in production.
Move content to /srv/gentoo. On GCP we need to explicitly declare a
tmpfs (we are advised not to re-use /dev/shm).
In the Dockerfile, explicitly make /srv/gentoo and try to make a tmpfs
there.
Sync directly from Turnstone, because its fast at syncing and more
reliable than rsync.us.gentoo.org.
Remove --checksum from args as it causes some mirrors to fail.
Log with logger -t so logs get sent to stackdriver on gcp.
Disable chroot; it doesn't seem to work with Containers on GCE.
Signed-off-by: Alec Warner <antarus@gentoo.org>
-rw-r--r-- | src/infra.gentoo.org/rsync-node/Dockerfile | 8 | ||||
-rw-r--r-- | src/infra.gentoo.org/rsync-node/rsyncd.conf | 2 | ||||
-rwxr-xr-x | src/infra.gentoo.org/rsync-node/wrap_rsync.sh | 79 |
3 files changed, 64 insertions, 25 deletions
diff --git a/src/infra.gentoo.org/rsync-node/Dockerfile b/src/infra.gentoo.org/rsync-node/Dockerfile index cd10282..7948ca8 100644 --- a/src/infra.gentoo.org/rsync-node/Dockerfile +++ b/src/infra.gentoo.org/rsync-node/Dockerfile @@ -8,12 +8,12 @@ COPY wrap_rsync.sh /opt/rsync/wrap_rsync.sh # docker build . --build_arg WAIT_TIME=30m -t gentoo/rsync # However, ARG's cannot be passed to ENTRYPOINTs, so we set these as ENV instead. # Mirror to get data from. -ENV SOURCE_MIRROR=rsync://boobie.gentoo.org/gentoo-portage +ENV SOURCE_MIRROR=rsync://turnstone.gentoo.org./gentoo-portage # ENV SOURCE_MIRROR=rsync://rsync.us.gentoo.org/gentoo-portage # Where to write the data in this container. -ENV DEST_DIR=/dev/shm/gentoo +ENV DEST_DIR=/srv/gentoo # How long to wait between syncs; must be a valid argument to sleep -ENV WAIT_TIME=30s +ENV WAIT_TIME=30m # This needs to exist in the container. WORKDIR $DEST_DIR @@ -21,4 +21,6 @@ WORKDIR $DEST_DIR # Expose Rsync port EXPOSE 873 +CMD mount -t tmpfs -o size=2g,nr_inodes=2000000 tmpfs /srv/gentoo + ENTRYPOINT /opt/rsync/wrap_rsync.sh diff --git a/src/infra.gentoo.org/rsync-node/rsyncd.conf b/src/infra.gentoo.org/rsync-node/rsyncd.conf index 7fbd4b9..0ff0a54 100644 --- a/src/infra.gentoo.org/rsync-node/rsyncd.conf +++ b/src/infra.gentoo.org/rsync-node/rsyncd.conf @@ -2,7 +2,7 @@ uid = nobody gid = nobody # We are in a container, who cares. -# use chroot = no +use chroot = no # Let clients use as much as they want; CPU control is in a load balancer in front of us. max connections = 0 diff --git a/src/infra.gentoo.org/rsync-node/wrap_rsync.sh b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh index b4a6857..a7f87ed 100755 --- a/src/infra.gentoo.org/rsync-node/wrap_rsync.sh +++ b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh @@ -4,6 +4,14 @@ # Then execute rsyncd; we will start serving once the sync completes. # Then keep syncing in the background every 30m. +# Maintain 2 'partitions' of the tree. +# "serving" - This copy is served to users and is not mutated. +# "updating" - This copy is a shadow copy used for updates. +# Create the two partitions on startup. +PARTITION1=$(mktemp -d -p "${DEST_DIR}" XXXXXX) +PARTITION2=$(mktemp -d -p "${DEST_DIR}" XXXXXX) + +# Function sync syncs dest ("${2}") from source ("${1}") function sync() { OPTS=( --quiet @@ -13,37 +21,66 @@ function sync() { --times --delete --timeout=300 - --checksum + --progress + # NOTE(antarus): Checksum upsets some public mirror nodes; so don't use it for now. + # --checksum ) SRC="${2}" DST="${1}" - echo "Started update at" $(date) >> $0.log 2>&1 + logger -t rsync "Started update at: $(date)" logger -t rsync "re-rsyncing the gentoo-portage tree" /usr/bin/rsync ${OPTS[@]} "${SRC}" "${DST}" >> $0.log 2>&1 - echo "End: "$(date) >> $0.log 2>&1 + err=$? + if [[ $err -ne 0 ]]; then + logger -t rsync "Failed to rsync tree: ${SRC}: $(date)" + return 1 + fi + logger -t rsync "End: $(date)" return 0 } -tmp=$(mktemp -d -p "${DEST_DIR}" XXXXXX) -sync "${tmp}" "${SOURCE_MIRROR}" # this is synchronous. +# Function init does a first sync, to populate the serving partition and +# setup symlinks, and begin serving data. +# "${1}" is the serving partition. "${2}" is the update partition +function init() { + sync "${1}" "${SOURCE_MIRROR}" # this is synchronous. -# We serve out of ${DEST_DIR}/serving -ln -s "${tmp}" "serving" + # We serve out of ${DEST_DIR}/serving + ln -s "${1}" "serving" + # Setup the update partition + ln -s "${2}" "update" -# Then launch rsyncd; it will detach into the background and serve from serving. -rsync --daemon --config="/opt/rsync/rsyncd.conf" + # Then launch rsyncd; it will detach into the background and serve from serving. + rsync --daemon --config="/opt/rsync/rsyncd.conf" +} -while true -do - sleep "${WAIT_TIME}" - tmp=$(mktemp -d -p "${DEST_DIR}" XXXXXX) - # If we fail to sync, just try again. - if ! sync "${tmp}" "${SOURCE_MIRROR}"; then - rm -rf "${tmp}" - continue +# Function update syncs the 'update' partition and, if successful, swaps the partitions. +function update() { + update=$(readlink "${DEST_DIR}/update") + # If we fail to sync, just return false and avoid modifying the serving state. + if ! sync "${update}" "${SOURCE_MIRROR}"; then + return 1 fi - # Atomically rename - ln -sf "${tmp}" "staging" && \ - mv -fT "${DEST_DIR}/staging" "${DEST_DIR}/serving" -done + + # Save the previous serving partition + old_serving=$(readlink "${DEST_DIR}/serving") + # Point the serving symlink at the update partition; now freshly updated. + mv -fT "${DEST_DIR}/update" "${DEST_DIR}/serving" + # Point the update partition at the old serving partition. + ln -sf "${old_serving}" "update" +} + +function serve() { + while true + do + # TODO(antarus): Add exponential backoff. + sleep "${WAIT_TIME}" + update + done +} + +# Partition1 starts as "serving", partition2 as "update" +init "${PARTITION1}" "${PARTITION2}" +# Serve forever +serve |