#!/bin/bash
# mirror-debian -- Daily mirror of the Debian, Ubuntu, and Auristor repositories.

# Updated with Claude Code assistance.

######################################################################
# Setup
######################################################################

# Enable verbose mode if -v was given on the command line.
if [[ "$1" = "-v" ]]; then
    verbose=-v
fi

# If DRYRUN is set, print the mirror commands instead of running them.
if [[ -n "$DRYRUN" ]]; then
    run=echo
else
    run=
fi

# Skip the lock file in dry-run mode so we don't block real runs.
if [[ -z "$run" ]]; then
    dotlockfile -l -r 12 -p /var/run/mirror2.pid || exit
fi

# Start the timer.
start_secs=$SECONDS

# Expect the trusted keyring in /var/lib/debmirror.
GNUPGHOME=/var/lib/debmirror; export GNUPGHOME

CONFIG_FILE=/var/lib/debmirror/config

# Load configuration. The config file is required.
if [[ ! -r "$CONFIG_FILE" ]]; then
    echo "$CONFIG_FILE not found or not readable; aborting" >&2
    exit 1
fi
. "$CONFIG_FILE" || { echo "$CONFIG_FILE exited with an error; aborting" >&2; exit 1; }

# ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## ####
missing_config() {
    echo "error: required configuration variable '$1' missing from $CONFIG_FILE; aborting" >&2
    exit 1
}
# ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## #### ## ####

# Make sure that all required configuration variables are defined.
[[ -z "$mirror_debian" ]]         && missing_config mirror_debian
[[ -z "$mirror_security" ]]       && missing_config mirror_security
[[ -z "$mirror_ubuntu" ]]         && missing_config mirror_ubuntu
[[ -z "$debian_dists" ]]          && missing_config debian_dists
[[ -z "$debian_security_dists" ]] && missing_config debian_security_dists
[[ -z "$ubuntu_dists" ]]          && missing_config ubuntu_dists
[[ -z "$archs_debian" ]]          && missing_config archs_debian
[[ -z "$archs_ubuntu" ]]          && missing_config archs_ubuntu
[[ -z "$archs_auristor" ]]        && missing_config archs_auristor

# rsync-options='-rltIL --partial --no-motd'
# -r = recurse into subdirectories
# -l = copy symlinks as symlinks
# -t = preserve modification times
# -I = don't skip files that match size and time
# -L = transform symlink into referent file/dir
# --partial = keep partially transferred files
# --no-motd = suppress daemon-mode MOTD

# We add the --ignore option so that debmirror does not attempt to cleanup
# files like ".nfs00000000000c3f71000003f4". These represent files that
# have been deleted but are still held open by some process. NFS will take
# care of cleaning up these files in its own time.

######################################################################
# Mirror the Debian repositories
######################################################################

# Mirror the usual repos, typically experimental, sid, and whatever is
# testing, stable, oldstable, and oldoldstable. These repos are
# defined as a comma-delimited list in the variable $debian_dists.
$run debmirror -a "$archs_debian" -e rsync --getcontents --precleanup --diff=none --i18n \
    $verbose --rsync-options='-rltIL --partial --no-motd' \
    --ignore='[.]nfs[[:xdigit:]]+' \
    -d "$debian_dists"             \
    -h "$mirror_debian" -r debian  \
    /srv/mirrors/debian

# Mirror the security updates repos, usually testing, stable, and oldstable
# (and sometimes oldoldstable).
$run debmirror -a "$archs_debian" -e rsync --precleanup --diff=none --i18n \
    $verbose --rsync-options='-rltIL --partial --no-motd' \
    --ignore='[.]nfs[[:xdigit:]]+' \
    -d "$debian_security_dists"    \
    -h "$mirror_security" -r debian-security \
    /srv/mirrors/debian-security

######################################################################
# Mirror the Ubuntu repositories
######################################################################

$run debmirror -a "$archs_ubuntu" -e rsync --getcontents --precleanup --diff=none --i18n \
    $verbose --rsync-options='-rltIL --partial --no-motd' \
    --ignore='[.]nfs[[:xdigit:]]+' \
    -d "$ubuntu_dists" -s main,multiverse,restricted,universe,main/debian-installer \
    -h "$mirror_ubuntu" -r ubuntu \
    /srv/mirrors/ubuntu

######################################################################
# Mirror the Auristor debian packages
######################################################################

AURISTOR_REPO_BASE_DIR=/srv/mirrors/auristor
mkdir -p "$AURISTOR_REPO_BASE_DIR"

# Set the Auristor client distributions you want to mirror in
# /var/lib/debmirror/config. Leave the auristor_codenames environment
# variable undefined to SKIP mirroring any Auristor clients.
#
# Example:
# auristor_codenames=("bullseye" "bookworm" "bionic")
#
# Due to the unusual directory format, to configure a Debian apt client
# to find the files you should add the following to an apt sources.list.d/ file:
#
#   deb [arch=amd64] https://debian.stanford.edu/auristor/bookworm bookworm client
#
# replacing "bookworm" TWICE in the above with whichever distribution you are using.

for codename in "${auristor_codenames[@]}"; do
    if [[ -n "$verbose" ]]; then
        echo "starting mirror of Auristor codename '$codename'"
    fi

    # We store each distribution in its own subdirectory of
    # AURISTOR_REPO_BASE_DIR. This is non-standard but that is how
    # Auristor has set up their repository.
    REPO_DIR="$AURISTOR_REPO_BASE_DIR/$codename"
    mkdir -p "$REPO_DIR"

    # Because of the unusual directory structure we have to mirror each
    # distribution separately. Thus we use --nocleanup, otherwise each
    # sync would start from scratch and mirror all files, even ones that
    # had been downloaded previously.
    #
    # Auristor's repository does not support rsync, so we turn off the initial rsync
    # file downloads by using the --rsync-extra=none option.
    #
    # Output and error lines to ignore:
    ignore="Warning: --rsync-extra is not configured to mirror the trace files"
    ignore="${ignore}|This configuration is not recommended"
    #
    # We filter out the ignore lines so they don't show up in root mail output.
    if [[ -n "$run" ]]; then
        echo debmirror $verbose -s client -a "$archs_auristor" --nocleanup --method=https --rsync-extra=none \
                  --getcontents --diff=none --i18n \
                  -d "${codename}" \
                  -h "client-rpm-repo.auristor.com" \
                  -r "filesystem/repo/recommended/${codename}" "$REPO_DIR"
    else
        debmirror $verbose -s client -a "$archs_auristor" --nocleanup --method=https --rsync-extra=none \
                  --getcontents --diff=none --i18n \
                  -d "${codename}" \
                  -h "client-rpm-repo.auristor.com" \
                  -r "filesystem/repo/recommended/${codename}" "$REPO_DIR" 2>&1 | grep -P -v "$ignore"
    fi
done

######################################################################
# Clean up
######################################################################

# Stop the timer and write elapsed time to the logfile for the record.
elapsed_secs=$(( SECONDS - start_secs ))
logfile='/var/lib/debmirror/repo_choices.log'
date=$(date --rfc-3339=seconds)
echo "[${date}] mirroring finished (elapsed seconds: $elapsed_secs)" >> "$logfile"

# Unlock the lock file (only if we acquired it).
if [[ -z "$run" ]]; then
    dotlockfile -u /var/run/mirror2.pid
fi

exit 0

# Can stop shellcheck warnings from this point on.
# shellcheck disable=all
DOCS=<<__END_OF_DOCS__

=for stopwords
Auristor Huaqing Zheng debmirror mirror-debian keyring DRYRUN

=head1 NAME

mirror-debian - Mirror Debian, Ubuntu, and Auristor repositories

=head1 SYNOPSIS

B<mirror-debian> [B<-v>]

=head1 DESCRIPTION

B<mirror-debian> creates a mirror of Debian, Ubuntu, and Auristor
repositories in F</srv/mirrors>.  It uses B<rsync> via B<debmirror>
for Debian and Ubuntu, and HTTPS for Auristor.

The architectures mirrored for each repository group are configured via
the variables C<archs_debian>, C<archs_ubuntu>, and C<archs_auristor>
in the configuration file.

B<mirror-debian> is normally run three times a day from cron.

=head1 OPTIONS

=over 4

=item B<-v>

Verbose mode.  Passes the C<-v> flag to B<debmirror>, which will then
print out what it is doing as it runs.

=back

=head1 ENVIRONMENT

=over 4

=item DRYRUN

If set to a non-empty value, the mirror commands are printed instead of
executed, and no lock file is acquired.

=back

=head1 FILES

=over 4

=item F</var/lib/debmirror/config>

Required configuration file.  The syntax should be a Bourne shell script
that sets shell variables.  The following variables must be set:

=over 4

=item mirror_debian

=item mirror_security

=item mirror_ubuntu

The host from which to mirror each distribution.  The Debian and Ubuntu
hosts must support rsync mirroring.

=item debian_dists

The comma-separated list of Debian distributions to mirror.

=item debian_security_dists

The comma-separated list of Debian security distributions to mirror.

=item ubuntu_dists

The comma-separated list of Ubuntu distributions to mirror.

=item archs_debian

=item archs_ubuntu

=item archs_auristor

The comma-separated list of architectures to mirror for each repository
group (e.g., C<amd64,i386>).

=back

The following variable is optional:

=over 4

=item auristor_codenames

A bash array of distribution codenames for which to mirror Auristor
client packages.  If not set, Auristor mirroring is skipped.

=back

=item F</var/lib/debmirror/trustedkeys.gpg>

Expects the trusted keyring for B<debmirror> to be found here.  This
keyring must include all of the keys for all the Release files of all the
distributions being mirrored.

=back

=head1 SEE ALSO

debmirror(1)

=head1 AUTHORS

Huaqing Zheng and Russ Allbery <rra@stanford.edu>.

=cut

__END_OF_DOCS__
