Various rsync node improvements.

WAIT_TIME=30m to test in production.

Move content to /srv/gentoo. On GCP we need to explicitly declare a
tmpfs (we are advised not to re-use /dev/shm).

In the Dockerfile, explicitly make /srv/gentoo and try to make a tmpfs
there.

Sync directly from Turnstone, because its fast at syncing and more
reliable than rsync.us.gentoo.org.

Remove --checksum from args as it causes some mirrors to fail.

Log with logger -t so logs get sent to stackdriver on gcp.

Disable chroot; it doesn't seem to work with Containers on GCE.

Signed-off-by: Alec Warner <antarus@gentoo.org>

3 files changed, 64 insertions, 25 deletions

diff --git a/src/infra.gentoo.org/rsync-node/Dockerfile b/src/infra.gentoo.org/rsync-node/Dockerfile
index cd10282..7948ca8 100644
--- a/src/infra.gentoo.org/rsync-node/Dockerfile
+++ b/src/infra.gentoo.org/rsync-node/Dockerfile
@@ -8,12 +8,12 @@ COPY wrap_rsync.sh /opt/rsync/wrap_rsync.sh
 # docker build . --build_arg WAIT_TIME=30m -t gentoo/rsync
 # However, ARG's cannot be passed to ENTRYPOINTs, so we set these as ENV instead.
 # Mirror to get data from.
-ENV SOURCE_MIRROR=rsync://boobie.gentoo.org/gentoo-portage
+ENV SOURCE_MIRROR=rsync://turnstone.gentoo.org./gentoo-portage
 # ENV SOURCE_MIRROR=rsync://rsync.us.gentoo.org/gentoo-portage
 # Where to write the data in this container.
-ENV DEST_DIR=/dev/shm/gentoo
+ENV DEST_DIR=/srv/gentoo
 # How long to wait between syncs; must be a valid argument to sleep
-ENV WAIT_TIME=30s
+ENV WAIT_TIME=30m
 
 # This needs to exist in the container.
 WORKDIR $DEST_DIR
@@ -21,4 +21,6 @@ WORKDIR $DEST_DIR
 # Expose Rsync port
 EXPOSE 873
 
+CMD mount -t tmpfs -o size=2g,nr_inodes=2000000 tmpfs /srv/gentoo
+
 ENTRYPOINT /opt/rsync/wrap_rsync.sh
diff --git a/src/infra.gentoo.org/rsync-node/rsyncd.conf b/src/infra.gentoo.org/rsync-node/rsyncd.conf
index 7fbd4b9..0ff0a54 100644
--- a/src/infra.gentoo.org/rsync-node/rsyncd.conf
+++ b/src/infra.gentoo.org/rsync-node/rsyncd.conf
@@ -2,7 +2,7 @@
 uid = nobody
 gid = nobody
 # We are in a container, who cares.
-# use chroot = no
+use chroot = no
 # Let clients use as much as they want; CPU control is in a load balancer in front of us.
 max connections = 0
 
diff --git a/src/infra.gentoo.org/rsync-node/wrap_rsync.sh b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh
index b4a6857..a7f87ed 100755
--- a/src/infra.gentoo.org/rsync-node/wrap_rsync.sh
+++ b/src/infra.gentoo.org/rsync-node/wrap_rsync.sh
@@ -4,6 +4,14 @@
 # Then execute rsyncd; we will start serving once the sync completes.
 # Then keep syncing in the background every 30m.
 
+# Maintain 2 'partitions' of the tree.
+# "serving" - This copy is served to users and is not mutated.
+# "updating" - This copy is a shadow copy used for updates.
+# Create the two partitions on startup.
+PARTITION1=$(mktemp -d -p "${DEST_DIR}" XXXXXX)
+PARTITION2=$(mktemp -d -p "${DEST_DIR}" XXXXXX)
+
+# Function sync syncs dest ("${2}") from source ("${1}")
 function sync() {
 	OPTS=(
 	 --quiet
@@ -13,37 +21,66 @@ function sync() {
 	 --times
 	 --delete
 	 --timeout=300
-	 --checksum
+	 --progress
+	 # NOTE(antarus): Checksum upsets some public mirror nodes; so don't use it for now.
+   #	 --checksum
 	)
 	SRC="${2}"
 	DST="${1}"
   
-	echo "Started update at" $(date) >> $0.log 2>&1
+	logger -t rsync "Started update at: $(date)"
 	logger -t rsync "re-rsyncing the gentoo-portage tree"
 	/usr/bin/rsync ${OPTS[@]} "${SRC}" "${DST}" >> $0.log 2>&1
-	echo "End: "$(date) >> $0.log 2>&1
+	err=$?
+	if [[ $err -ne 0 ]]; then
+		logger -t rsync "Failed to rsync tree: ${SRC}: $(date)"
+		return 1
+	fi
+	logger -t rsync "End: $(date)"
 	return 0
 }
 
-tmp=$(mktemp -d -p "${DEST_DIR}" XXXXXX)
-sync "${tmp}" "${SOURCE_MIRROR}" # this is synchronous.
+# Function init does a first sync, to populate the serving partition and
+# setup symlinks, and begin serving data.
+# "${1}" is the serving partition. "${2}" is the update partition
+function init() {
+	sync "${1}" "${SOURCE_MIRROR}" # this is synchronous.
 
-# We serve out of ${DEST_DIR}/serving
-ln -s "${tmp}" "serving"
+	# We serve out of ${DEST_DIR}/serving
+	ln -s "${1}" "serving"
+	# Setup the update partition
+	ln -s "${2}" "update"
 
-# Then launch rsyncd; it will detach into the background and serve from serving.
-rsync --daemon --config="/opt/rsync/rsyncd.conf"
+	# Then launch rsyncd; it will detach into the background and serve from serving.
+	rsync --daemon --config="/opt/rsync/rsyncd.conf"
+}
 
-while true
-do
-	sleep "${WAIT_TIME}"
-	tmp=$(mktemp -d -p "${DEST_DIR}" XXXXXX)
-	# If we fail to sync, just try again.
-	if ! sync "${tmp}" "${SOURCE_MIRROR}"; then
-		rm -rf "${tmp}"
-		continue
+# Function update syncs the 'update' partition and, if successful, swaps the partitions.
+function update() {
+	update=$(readlink "${DEST_DIR}/update")
+	# If we fail to sync, just return false and avoid modifying the serving state.
+	if ! sync "${update}" "${SOURCE_MIRROR}"; then
+		return 1
 	fi
-	# Atomically rename
-	ln -sf "${tmp}" "staging" && \
-		mv -fT "${DEST_DIR}/staging" "${DEST_DIR}/serving"
-done
+
+	# Save the previous serving partition
+	old_serving=$(readlink "${DEST_DIR}/serving")
+	# Point the serving symlink at the update partition; now freshly updated.
+	mv -fT "${DEST_DIR}/update" "${DEST_DIR}/serving"
+	# Point the update partition at the old serving partition.
+	ln -sf "${old_serving}" "update"
+}
+
+function serve() {
+	while true
+	do
+		# TODO(antarus): Add exponential backoff.
+		sleep "${WAIT_TIME}"
+		update
+	done
+}
+
+# Partition1 starts as "serving", partition2 as "update"
+init "${PARTITION1}" "${PARTITION2}"
+# Serve forever
+serve