diff options
author | Alec Warner <antarus@gentoo.org> | 2020-04-23 16:33:39 -0700 |
---|---|---|
committer | Alec Warner <antarus@gentoo.org> | 2020-04-23 16:33:39 -0700 |
commit | e91c97ff20233d2e0048e462ad25be22d2123edb (patch) | |
tree | 33f1f284c9ea45384bb2cf477925a12c2a97d30c | |
download | rsync-service-e91c97ff20233d2e0048e462ad25be22d2123edb.tar.gz rsync-service-e91c97ff20233d2e0048e462ad25be22d2123edb.tar.bz2 rsync-service-e91c97ff20233d2e0048e462ad25be22d2123edb.zip |
Add a docker container for rsync.
This is cribbed off of my old work at
https://gitweb.gentoo.org/dev/antarus.git/tree/src/infra.gentoo.org/rsync-node.
Basically its a stage3 that serves rsync atomically. It needs some tests
and some CI.
Signed-off-by: Alec Warner <antarus@gentoo.org>
-rw-r--r-- | Dockerfile | 36 | ||||
-rw-r--r-- | rsync-mirror | 22 | ||||
-rwxr-xr-x | wrap_rsync.sh | 109 |
3 files changed, 167 insertions, 0 deletions
diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..73a69da --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Base Image +FROM gentoo/stage3-amd64-hardened:latest AS base +WORKDIR / + +COPY wrap_rsync.sh /opt/rsync/wrap_rsync.sh +COPY ./rsync-mirror /etc/xinetd.d/rsync-mirror + +# Normally I would advocate for ARG here and pass arguments to wrap_rsync. +# This would enable new docker builds with arguments like: +# docker build . --build_arg WAIT_TIME=30m -t gentoo/rsync +# However, ARG's cannot be passed to ENTRYPOINTs, so we set these as ENV instead. + +# Mirror to get data from. +ENV SOURCE_MIRROR=rsync://turnstone.gentoo.org./gentoo-portage +# ENV SOURCE_MIRROR=rsync://rsync.us.gentoo.org/gentoo-portage + +# Possibly a stateful volume +ENV DEST_DIR=/srv/gentoo +# A memory-backed volume +ENV TMP_DIR=/srv/ephemeral + +# How long to wait between syncs; must be a valid argument to sleep +ENV WAIT_TIME=30m + +# Create TMP_DIR and DEST_DIR +WORKDIR $TMP_DIR +WORKDIR $DEST_DIR + +# Expose Rsync port +EXPOSE 873 + +# Stop xinetd; wrap_rsync will start it when the container is started. +CMD /etc/init.d/xinetd stop + +# Execute wrapper. +ENTRYPOINT /opt/rsync/wrap_rsync.sh diff --git a/rsync-mirror b/rsync-mirror new file mode 100644 index 0000000..18ffadc --- /dev/null +++ b/rsync-mirror @@ -0,0 +1,22 @@ +# rsync.gentoo.org service + +service rsync +{ + socket_type = stream + wait = no + user = root + server = /usr/bin/rsync + server_args = --daemon + log_on_success = PID HOST DURATION EXIT + log_on_failure = HOST ATTEMPT + log_type = SYSLOG local3 + port = rsync + only_from = 0.0.0.0/0 + nice = 5 + instances = 60 + cps = 500 1 + max_load = 4 + per_source = 10 + flags = IPv4 + disable = no +} diff --git a/wrap_rsync.sh b/wrap_rsync.sh new file mode 100755 index 0000000..fb420dc --- /dev/null +++ b/wrap_rsync.sh @@ -0,0 +1,109 @@ +#!/bin/bash + +set -x + +# On container start, run an rsync to get a good copy of the tree. +# Once we have a copy, start xinetd to start serving. +# Then keep syncing in the background every 30m. + +# Maintain 2 'partitions' of the tree. +# "serving" - This copy is served to users and is not mutated. +# "updating" - This copy is a shadow copy used for updates. +# Create the two partitions on startup. +# We will swap between them at runtime. + +PARTITION1=$(mktemp -d -p "${DEST_DIR}" XXXXXX) +PARTITION2=$(mktemp -d -p "${DEST_DIR}" XXXXXX) +# Our stateful copy. +UPDATES=${DEST_DIR}/stateful + +# Function sync syncs dest ("${2}") from source ("${1}") +function sync() { + OPTS=( + --quiet + --recursive + --links + --perms + --times + --delete + --timeout=300 + --progress + # NOTE(antarus): Checksum upsets some public mirror nodes; so don't use it for now. + # --checksum + ) + SRC="${2}" + DST="${1}" + + logger -t rsync "Started update at: $(date)" + logger -t rsync "re-rsyncing the gentoo-portage tree" + /usr/bin/rsync ${OPTS[@]} "${SRC}" "${DST}" >> $0.log 2>&1 + err=$? + if [[ $err -ne 0 ]]; then + logger -t rsync "Failed to rsync tree: ${SRC}: $(date)" + return 1 + fi + logger -t rsync "End: $(date)" + return 0 +} + +# Init will update the stateful tree copy. +# Then it will copy that to the stateless partition. +function init() { + sync "${UPDATING}" "${SOURCE_MIRROR}" # this is synchronous. + sync "${UPDATING}" "${PARTITION1}" + + # We serve out of ${TMP_DIR}/serving + ln -s "${PARTITION1}" "${TMP_DIR}/serving" + # The second partition will be for stateless updates. + ln -s "${PARTITION2}" "${TMP_DIR}/update" + + # Then launch xinetd; it will detech into the background and serve. + /etc/init.d/xinetd start +} + +# function update syncs the UPDATING partition (stateful.) +# Then it syncs it into the ${TMP_DIR}/update partition (stateless.) +# Then we swap the two stateless partitions (via symlinks swapping.) +function update() { + update=$(readlink "${TMP_DIR}/update") + # Try to update our stateful tree copy. + if ! sync "${UPDATING}" "${SOURCE_MIRROR}"; then + return 1 + fi + + # Try to update our stateless copy. + if ! sync "${update}" "${UPDATING}"; then + return 1 + fi + + # Quasi-atomic swap with symlinks. + # Save the previous serving partition + old_serving=$(readlink "${TMP_DIR}/serving") + # Point the serving symlink at the update partition; now freshly updated. + mv -fT "${TMP_DIR}/update" "${TMP_DIR}/serving" + # Point the update partition at the old serving partition. + ln -sf "${old_serving}" "${TMP_DIR}/update" + + # Its plausible here that users may still be accessing the old_serving copy, so we don't delete it or anything. +} + +function health() { + /etc/init.d/xinetd status +} + +function serve() { + while true + do + sleep "${WAIT_TIME}" + update + # If xinetd died, just suicide and docker will restart us. + health || exit 5 + done +} + +init +if [[ $? -ne 0 ]]; then + exit 1 +fi +# Serve forever +serve |