diff --git a/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml b/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml index 5f0617bc..a362a9a6 100644 --- a/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml +++ b/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml @@ -31,7 +31,7 @@ nomad_host_volumes: path: /data/void-packages read_only: false - name: root-pkgs - path: /hostdir/binpkgs + path: /srv/www/void-repo/current read_only: false - name: root_mirror path: /srv/www/void-repo @@ -48,3 +48,6 @@ nomad_host_volumes: - name: ccache path: /hostdir/ccache read_only: true + - name: incoming_pkgs + path: /incoming + read_only: false diff --git a/ansible/host_vars/a-sea-us.m.voidlinux.org.yml b/ansible/host_vars/a-sea-us.m.voidlinux.org.yml index fdbb5580..477757b0 100644 --- a/ansible/host_vars/a-sea-us.m.voidlinux.org.yml +++ b/ansible/host_vars/a-sea-us.m.voidlinux.org.yml @@ -8,3 +8,24 @@ xbps_repository_nonfree: https://{{ xbps_repository_address }}/current/aarch64/n xbps_repoconf_repos_enabled: - 00-repository-main - 10-repository-nonfree + +sshd_AllowGroups: + - build-ops + +buildworker_archs: + - aarch64 + - aarch64-musl + +nomad_host_volumes: + - name: ccache + path: /hostdir/ccache + read_only: true + - name: aarch64_hostdir + path: /hostdir + read_only: false + - name: aarch64_workdir + path: /builder/workdir + read_only: false + - name: aarch64_buildrootdir + path: /builder/buildroot + read_only: false diff --git a/ansible/inventory b/ansible/inventory index d661ecf6..40f73d18 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -2,6 +2,7 @@ a-hel-fi.m.voidlinux.org a-fsn-de.m.voidlinux.org b-fsn-de.m.voidlinux.org +a-sea-us.m.voidlinux.org [root_mirror] a-fsn-de.m.voidlinux.org diff --git a/services/nomad/build/build-rsyncd.nomad b/services/nomad/build/build-rsyncd.nomad index f9c3e300..6e07a306 100644 --- a/services/nomad/build/build-rsyncd.nomad +++ b/services/nomad/build/build-rsyncd.nomad @@ -9,7 +9,7 @@ job "build-rsyncd" { mode = "bridge" port "rsync" { to = 873 - host_network = "internal" + # XXX: host_network = "internal" } } @@ -19,6 +19,18 @@ job "build-rsyncd" { read_only = false } + volume "root_mirror" { + type = "host" + source = "root_mirror" + read_only = false + } + + volume "incoming_pkgs" { + type = "host" + source = "incoming_pkgs" + read_only = false + } + service { provider = "nomad" name = "build-rsyncd" @@ -29,8 +41,7 @@ job "build-rsyncd" { driver = "docker" config { - image = "ghcr.io/void-linux/infra-rsync:20240709R1" - volumes = [ "local/buildsync.conf:/etc/rsyncd.conf.d/buildsync.conf" ] + image = "ghcr.io/void-linux/infra-rsync:20251102R1" } resources { @@ -43,17 +54,26 @@ job "build-rsyncd" { destination = "/hostdir" } + volume_mount { + volume = "root_mirror" + destination = "/mirror" + } + + volume_mount { + volume = "incoming_pkgs" + destination = "/incoming" + } + template { - data = file("xbps-clean-sigs") - destination = "local/xbps-clean-sigs" + data = file("rsync-post-xfer") + destination = "local/rsync-post-xfer" perms = "0755" } template { data = < Syncing packages to the shadow repository..." +rsync -vurk --delete-after \ +--filter='+ */' --filter='+ *.%(prop:target)s.xbps' \ +--filter='- .*' --filter='- *' \ --password-file=/secrets/rsync/password /hostdir/binpkgs/ \ {{ range nomadService 1 $allocID "build-rsyncd" -}} -rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s -{{ end -}}""")] +rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s +{{ end -}} +echo "=> Cleaning built packages..." +find /hostdir/binpkgs -name '*.xbps' -o -name '*-repodata' -print -delete +""")] factory.addStep(GitWithDiff( - repourl='https://github.com/void-linux/void-packages.git', + # XXX + repourl='https://github.com/classabbyamp/void-packages.git', mode='incremental', workdir=distdir(''), progress=True, @@ -271,7 +267,8 @@ factory.addStep(GitWithDiff( )) factory.addStep(steps.Git( - repourl='https://github.com/void-linux/xbps-bulk.git', + # XXX + repourl='https://github.com/classabbyamp/xbps-bulk.git', mode='incremental', workdir=bulkdir(''), progress=True, @@ -350,18 +347,6 @@ factory.addStep(steps.ShellSequence( timeout=14400, )) -factory.addStep(steps.ShellCommand( - command=make_prune_cmd, - name='prune_packages', - description='removing obsolete packages', - descriptionDone='removed obsolete packages', - haltOnFailure=True, - logEnviron=False, - usePTY=True, - workdir='.', - timeout=14400, -)) - factory.addStep(steps.ShellCommand( command=make_rsync_cmd, name='sync_packages', @@ -371,8 +356,6 @@ factory.addStep(steps.ShellCommand( logEnviron=False, usePTY=True, workdir='.', - doStepIf=do_sync(), - hideStepIf=hide_skipped, timeout=14400, decodeRC={ 0: SUCCESS, @@ -393,8 +376,7 @@ for b in builders: 'target': targetarch, 'cross': str(hostarch != targetarch), 'worker': b['worker'], - 'sync': str(b['sync']), - 'bootstrap_args': b.get('bootstrap_args', '-N'), + 'bootstrap_args': b.get('bootstrap_args', ''), } c['builders'].append(util.BuilderConfig( diff --git a/services/nomad/build/buildbot.nomad b/services/nomad/build/buildbot.nomad index 3b7e5d25..eb4fccf6 100644 --- a/services/nomad/build/buildbot.nomad +++ b/services/nomad/build/buildbot.nomad @@ -17,7 +17,7 @@ job "buildbot" { port "worker" { to = 42042 - host_network = "internal" + # XXX: host_network = "internal" } } @@ -33,7 +33,9 @@ job "buildbot" { source = "netauth_config" } + # XXX service { + provider = "nomad" name = "buildbot-www" port = "http" @@ -46,12 +48,16 @@ job "buildbot" { } } + # XXX service { + provider = "nomad" name = "buildbot-worker" port = "worker" } + # XXX service { + provider = "nomad" name = "buildbot-metrics" port = "metrics" } @@ -84,17 +90,18 @@ job "buildbot" { read_only = true } + # XXX template { data = <"/hostdir/binpkgs/$dir/otime" - done - sleep 60 + t="$(date +%s)" + for dir in / /nonfree /debug /multilib /multilib/nonfree /nonfree; do + [ -e "/mirror/current/$dir" ] && echo "$t">"/mirror/current/$dir/otime" + [ -e "/mirror/current/musl/$dir" ] && echo "$t">"/mirror/current/musl/$dir/otime" + [ -e "/mirror/current/aarch64/$dir" ] && echo "$t">"/mirror/current/aarch64/$dir/otime" + done + sleep 60 done EOF - destination = "local/run.sh" - perms = "0755" - } + destination = "local/run.sh" + perms = "0755" } } } diff --git a/services/nomad/build/xbps-clean-sigs b/services/nomad/build/xbps-clean-sigs deleted file mode 100644 index 85c8f3b0..00000000 --- a/services/nomad/build/xbps-clean-sigs +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -# Remove signatures that don't have a corresponding package. -find /hostdir/binpkgs \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} + - -exit 0