From ccc0c84fee14d4eae086f8d1cd9d4dd5903ee48f Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Mon, 24 Nov 2025 23:31:58 -0500 Subject: [PATCH 1/8] ansible: add a-sea-us to buildworkers, add host vars/volumes for building --- .../host_vars/a-sea-us.m.voidlinux.org.yml | 21 +++++++++++++++++++ ansible/inventory | 1 + 2 files changed, 22 insertions(+) diff --git a/ansible/host_vars/a-sea-us.m.voidlinux.org.yml b/ansible/host_vars/a-sea-us.m.voidlinux.org.yml index fdbb5580..477757b0 100644 --- a/ansible/host_vars/a-sea-us.m.voidlinux.org.yml +++ b/ansible/host_vars/a-sea-us.m.voidlinux.org.yml @@ -8,3 +8,24 @@ xbps_repository_nonfree: https://{{ xbps_repository_address }}/current/aarch64/n xbps_repoconf_repos_enabled: - 00-repository-main - 10-repository-nonfree + +sshd_AllowGroups: + - build-ops + +buildworker_archs: + - aarch64 + - aarch64-musl + +nomad_host_volumes: + - name: ccache + path: /hostdir/ccache + read_only: true + - name: aarch64_hostdir + path: /hostdir + read_only: false + - name: aarch64_workdir + path: /builder/workdir + read_only: false + - name: aarch64_buildrootdir + path: /builder/buildroot + read_only: false diff --git a/ansible/inventory b/ansible/inventory index d661ecf6..40f73d18 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -2,6 +2,7 @@ a-hel-fi.m.voidlinux.org a-fsn-de.m.voidlinux.org b-fsn-de.m.voidlinux.org +a-sea-us.m.voidlinux.org [root_mirror] a-fsn-de.m.voidlinux.org From b97b20b5434c794e5985f18167bae59c8bc9823f Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Mon, 24 Nov 2025 23:32:32 -0500 Subject: [PATCH 2/8] ansible: add incoming_pkgs volume, update root-pkgs path for a-fsn-de --- ansible/host_vars/a-fsn-de.m.voidlinux.org.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml b/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml index 5f0617bc..a362a9a6 100644 --- a/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml +++ b/ansible/host_vars/a-fsn-de.m.voidlinux.org.yml @@ -31,7 +31,7 @@ nomad_host_volumes: path: /data/void-packages read_only: false - name: root-pkgs - path: /hostdir/binpkgs + path: /srv/www/void-repo/current read_only: false - name: root_mirror path: /srv/www/void-repo @@ -48,3 +48,6 @@ nomad_host_volumes: - name: ccache path: /hostdir/ccache read_only: true + - name: incoming_pkgs + path: /incoming + read_only: false From 38db3b0b34a80fddc8ad8343ac6d869874459e81 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Fri, 25 Jul 2025 21:27:00 -0400 Subject: [PATCH 3/8] services/nomad/build/buildbot: native aarch64 worker, use remote repo (shadow) only --- services/nomad/build/buildbot.cfg | 70 +++++++++++------------------ services/nomad/build/buildbot.nomad | 18 ++++---- 2 files changed, 34 insertions(+), 54 deletions(-) diff --git a/services/nomad/build/buildbot.cfg b/services/nomad/build/buildbot.cfg index 83f80d4c..e996c764 100644 --- a/services/nomad/build/buildbot.cfg +++ b/services/nomad/build/buildbot.cfg @@ -10,7 +10,7 @@ from pathlib import Path from twisted.internet import defer from twisted.python import log -from buildbot.process.results import SUCCESS, SKIPPED +from buildbot.process.results import SUCCESS from buildbot.plugins import util, secrets, reporters, worker, schedulers from buildbot.plugins import steps @@ -129,13 +129,19 @@ c['schedulers'].append(schedulers.ForceScheduler( # ###### BUILDERS -distdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}') -bulkdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}') +def distdir(f): + return util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}') + +def bulkdir(f): + return util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}') + hostdir = '/hostdir' -buildroot = lambda: util.Interpolate(f'/buildroots/%(prop:name)s/buildroot') -builddir = lambda: util.Interpolate('builddir-%(prop:buildnumber)s') -do_sync = lambda: util.Interpolate('%(prop:sync)s') == "True" -hide_skipped = lambda results, _: results == SKIPPED + +def buildroot(): + return util.Interpolate('/buildroots/%(prop:name)s/buildroot') + +def builddir(): + return util.Interpolate('builddir-%(prop:buildnumber)s') factory = util.BuildFactory() @@ -198,7 +204,7 @@ def make_xbps_bulk_cmd(props): '-h', hostdir, '-d', distdir(''), '-m', buildroot(), - '-t', + '-t', '-s', '-S', ] if props.getProperty('cross') == 'True': command += ['-a', props.getProperty('target')] @@ -229,32 +235,21 @@ def build_packages(props): return cmds -@util.renderer -def make_prune_cmd(props): - return ['bash', '-c', - util.Interpolate(f""" -export XBPS_TARGET_ARCH="%(prop:target)s" -for repo in / /debug /nonfree /bootstrap; do - xbps-rindex -r "{hostdir}/binpkgs/$repo" -done -if [ "$XBPS_TARGET_ARCH" = i686 ]; then - for repo in /multilib /multilib/nonfree /multilib/bootstrap; do - XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo" - done -fi -""")] - @util.renderer def make_rsync_cmd(props): - return ['bash', '-c', + return ['bash', '-e', '-c', util.Interpolate(""" -rsync -vurk --delete-after --delay-updates \ ---filter='+ */' --filter='+ %(prop:target)s-repodata' --filter='+ *.%(prop:target)s.xbps' \ ---filter='+ otime' --filter='- .*' --filter='- *' \ +echo "=> Syncing packages to the shadow repository..." +rsync -vurk --delete-after \ +--filter='+ */' --filter='+ *.%(prop:target)s.xbps' \ +--filter='- .*' --filter='- *' \ --password-file=/secrets/rsync/password /hostdir/binpkgs/ \ {{ range nomadService 1 $allocID "build-rsyncd" -}} -rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s -{{ end -}}""")] +rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s +{{ end -}} +echo "=> Cleaning built packages..." +find /hostdir/binpkgs -name '*.xbps' -o -name '*-repodata' -print -delete +""")] factory.addStep(GitWithDiff( @@ -350,18 +345,6 @@ factory.addStep(steps.ShellSequence( timeout=14400, )) -factory.addStep(steps.ShellCommand( - command=make_prune_cmd, - name='prune_packages', - description='removing obsolete packages', - descriptionDone='removed obsolete packages', - haltOnFailure=True, - logEnviron=False, - usePTY=True, - workdir='.', - timeout=14400, -)) - factory.addStep(steps.ShellCommand( command=make_rsync_cmd, name='sync_packages', @@ -371,8 +354,6 @@ factory.addStep(steps.ShellCommand( logEnviron=False, usePTY=True, workdir='.', - doStepIf=do_sync(), - hideStepIf=hide_skipped, timeout=14400, decodeRC={ 0: SUCCESS, @@ -393,8 +374,7 @@ for b in builders: 'target': targetarch, 'cross': str(hostarch != targetarch), 'worker': b['worker'], - 'sync': str(b['sync']), - 'bootstrap_args': b.get('bootstrap_args', '-N'), + 'bootstrap_args': b.get('bootstrap_args', ''), } c['builders'].append(util.BuilderConfig( diff --git a/services/nomad/build/buildbot.nomad b/services/nomad/build/buildbot.nomad index 3b7e5d25..85524a72 100644 --- a/services/nomad/build/buildbot.nomad +++ b/services/nomad/build/buildbot.nomad @@ -107,15 +107,15 @@ EOF { name = "aarch64", max-builds = 2 }, ], builders = [ - { name = "x86_64", host = "x86_64", worker = "glibc", sync = false }, - { name = "i686", host = "i686", worker = "glibc", sync = false }, - { name = "armv7l", host = "x86_64", target = "armv7l", worker = "glibc", sync = false }, - { name = "armv6l", host = "x86_64", target = "armv6l", worker = "glibc", sync = false }, - { name = "x86_64-musl", host = "x86_64-musl", worker = "musl", sync = true }, - { name = "armv7l-musl", host = "x86_64-musl", target = "armv7l-musl", worker = "musl", sync = true }, - { name = "armv6l-musl", host = "x86_64-musl", target = "armv6l-musl", worker = "musl", sync = true }, - { name = "aarch64", host = "x86_64", target = "aarch64", worker = "aarch64", sync = true, bootstrap_args = "" }, - { name = "aarch64-musl", host = "x86_64-musl", target = "aarch64-musl", worker = "aarch64", sync = true, bootstrap_args = "" }, + { name = "x86_64", host = "x86_64", worker = "glibc", }, + { name = "i686", host = "i686", worker = "glibc", }, + { name = "armv7l", host = "x86_64", target = "armv7l", worker = "glibc", }, + { name = "armv6l", host = "x86_64", target = "armv6l", worker = "glibc", }, + { name = "x86_64-musl", host = "x86_64-musl", worker = "musl", }, + { name = "armv7l-musl", host = "x86_64-musl", target = "armv7l-musl", worker = "musl", }, + { name = "armv6l-musl", host = "x86_64-musl", target = "armv6l-musl", worker = "musl", }, + { name = "aarch64", host = "aarch64", worker = "aarch64" }, + { name = "aarch64-musl", host = "aarch64-musl", worker = "aarch64" }, ], }) destination = "local/workers.json" From 82a751229230df378f60dcaedcbcf0646bbb4923 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Fri, 25 Jul 2025 21:27:46 -0400 Subject: [PATCH 4/8] services/nomad/build/buildbot-worker: native aarch64 worker, use remote repo (shadow) only --- services/nomad/build/buildbot-worker.nomad | 25 +++++++++++----------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/services/nomad/build/buildbot-worker.nomad b/services/nomad/build/buildbot-worker.nomad index 1f866cb1..a2687734 100644 --- a/services/nomad/build/buildbot-worker.nomad +++ b/services/nomad/build/buildbot-worker.nomad @@ -10,7 +10,7 @@ job "buildbot-worker" { // memory_max is ~95% of capacity { name = "glibc", jobs = 10, cpu = 38100, mem = 115840, mem_max = 122270 }, { name = "musl", jobs = 6, cpu = 21700, mem = 57690, mem_max = 60890 }, - { name = "aarch64", jobs = 6, cpu = 12000, mem = 28500, mem_max = 30500 }, + { name = "aarch64", jobs = 12, cpu = 16000, mem = 27500, mem_max = 29500 }, ] labels = [ "buildbot-worker-${group.value.name}" ] @@ -161,21 +161,20 @@ EOF } // the builders should use local repos - // except for aarch64, which must be able to get hostmakedepends from repo-default template { data = < Date: Tue, 25 Nov 2025 01:42:19 -0500 Subject: [PATCH 5/8] services/nomad/build/timefiles: only run on shadow now that shadow is the first place a repository is assembled, this shouldn't run on builders --- services/nomad/build/timefiles.nomad | 66 ++++++++++++---------------- 1 file changed, 29 insertions(+), 37 deletions(-) diff --git a/services/nomad/build/timefiles.nomad b/services/nomad/build/timefiles.nomad index c31c4258..d0b3436f 100644 --- a/services/nomad/build/timefiles.nomad +++ b/services/nomad/build/timefiles.nomad @@ -3,53 +3,45 @@ job "timefiles" { datacenters = ["VOID"] namespace = "build" - dynamic "group" { - for_each = [ "glibc", "aarch64", "musl", ] - labels = [ "timefiles-${group.value}" ] - - content { - count = 1 - network { mode = "bridge" } + group "timefiles" { + count = 1 + network { mode = "bridge" } + + volume "root_mirror" { + type = "host" + source = "root_mirror" + read_only = false + } - dynamic "volume" { - for_each = [ "${group.value}" ] - labels = [ "${volume.value}_hostdir" ] + task "timefiles" { + driver = "docker" - content { - type = "host" - source = "${volume.value}_hostdir" - read_only = false - } + config { + image = "ghcr.io/void-linux/void-glibc:20240526R1" + command = "/local/run.sh" } - task "timefiles" { - driver = "docker" - - config { - image = "ghcr.io/void-linux/void-glibc:20240526R1" - command = "/local/run.sh" - } - - volume_mount { - volume = "${group.value}_hostdir" - destination = "/hostdir" - } + volume_mount { + volume = "root_mirror" + destination = "/mirror" + } - template { - data = <"/hostdir/binpkgs/$dir/otime" - done - sleep 60 + t="$(date +%s)" + for dir in / /nonfree /debug /multilib /multilib/nonfree /nonfree; do + [ -e "/mirror/current/$dir" ] && echo "$t">"/mirror/current/$dir/otime" + [ -e "/mirror/current/musl/$dir" ] && echo "$t">"/mirror/current/musl/$dir/otime" + [ -e "/mirror/current/aarch64/$dir" ] && echo "$t">"/mirror/current/aarch64/$dir/otime" + done + sleep 60 done EOF - destination = "local/run.sh" - perms = "0755" - } + destination = "local/run.sh" + perms = "0755" } } } From 96f551401238f0c2b4c7623083052fa2e91c11bc Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Tue, 25 Nov 2025 01:43:44 -0500 Subject: [PATCH 6/8] services/nomad/build/root-pkgs-httpd: update to 1.37 --- services/nomad/build/root-pkgs-httpd.nomad | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/nomad/build/root-pkgs-httpd.nomad b/services/nomad/build/root-pkgs-httpd.nomad index 87766f69..fd1aa1b4 100644 --- a/services/nomad/build/root-pkgs-httpd.nomad +++ b/services/nomad/build/root-pkgs-httpd.nomad @@ -27,7 +27,7 @@ job "build-mirror" { driver = "docker" config { - image = "busybox:1.32-musl" + image = "busybox:1.37-musl" args = ["httpd", "-f", "-p", "80", "-h", "/pkgs"] network_mode = "host" } From c7b6f4e32771abf3d82c13372f53a95ef7964565 Mon Sep 17 00:00:00 2001 From: classabbyamp Date: Tue, 25 Nov 2025 01:50:51 -0500 Subject: [PATCH 7/8] WIP: services/nomad/build/build-rsyncd: reorganise sync architecture now packages get synced to an incoming directory. a script runs after each rsync to move and register the packages into the repo --- services/nomad/build/build-rsyncd.nomad | 65 ++++++++++++++++++------- services/nomad/build/rsync-post-xfer | 46 +++++++++++++++++ services/nomad/build/xbps-clean-sigs | 6 --- 3 files changed, 93 insertions(+), 24 deletions(-) create mode 100644 services/nomad/build/rsync-post-xfer delete mode 100644 services/nomad/build/xbps-clean-sigs diff --git a/services/nomad/build/build-rsyncd.nomad b/services/nomad/build/build-rsyncd.nomad index f9c3e300..5f14be00 100644 --- a/services/nomad/build/build-rsyncd.nomad +++ b/services/nomad/build/build-rsyncd.nomad @@ -19,6 +19,18 @@ job "build-rsyncd" { read_only = false } + volume "root_mirror" { + type = "host" + source = "root_mirror" + read_only = false + } + + volume "incoming_pkgs" { + type = "host" + source = "incoming_pkgs" + read_only = false + } + service { provider = "nomad" name = "build-rsyncd" @@ -29,8 +41,7 @@ job "build-rsyncd" { driver = "docker" config { - image = "ghcr.io/void-linux/infra-rsync:20240709R1" - volumes = [ "local/buildsync.conf:/etc/rsyncd.conf.d/buildsync.conf" ] + image = "ghcr.io/void-linux/infra-rsync:20251102R1" } resources { @@ -43,17 +54,26 @@ job "build-rsyncd" { destination = "/hostdir" } + volume_mount { + volume = "root_mirror" + destination = "/mirror" + } + + volume_mount { + volume = "incoming_pkgs" + destination = "/incoming" + } + template { - data = file("xbps-clean-sigs") - destination = "local/xbps-clean-sigs" + data = file("rsync-post-xfer") + destination = "local/rsync-post-xfer" perms = "0755" } template { data = < Date: Tue, 25 Nov 2025 01:52:21 -0500 Subject: [PATCH 8/8] TEMP: for local testing, REMOVE ME --- services/nomad/build/build-rsyncd.nomad | 2 +- services/nomad/build/buildbot-worker.nomad | 19 ++++++++++----- services/nomad/build/buildbot.cfg | 6 +++-- services/nomad/build/buildbot.nomad | 27 ++++++++++++++-------- services/nomad/build/root-pkgs-httpd.nomad | 1 + 5 files changed, 37 insertions(+), 18 deletions(-) diff --git a/services/nomad/build/build-rsyncd.nomad b/services/nomad/build/build-rsyncd.nomad index 5f14be00..6e07a306 100644 --- a/services/nomad/build/build-rsyncd.nomad +++ b/services/nomad/build/build-rsyncd.nomad @@ -9,7 +9,7 @@ job "build-rsyncd" { mode = "bridge" port "rsync" { to = 873 - host_network = "internal" + # XXX: host_network = "internal" } } diff --git a/services/nomad/build/buildbot-worker.nomad b/services/nomad/build/buildbot-worker.nomad index a2687734..fd35db9a 100644 --- a/services/nomad/build/buildbot-worker.nomad +++ b/services/nomad/build/buildbot-worker.nomad @@ -8,9 +8,13 @@ job "buildbot-worker" { // cpu is ~equivalent to a number of cores // memory is ~90% of capacity // memory_max is ~95% of capacity - { name = "glibc", jobs = 10, cpu = 38100, mem = 115840, mem_max = 122270 }, - { name = "musl", jobs = 6, cpu = 21700, mem = 57690, mem_max = 60890 }, - { name = "aarch64", jobs = 12, cpu = 16000, mem = 27500, mem_max = 29500 }, + # XXX + # { name = "glibc", jobs = 10, cpu = 38100, mem = 115840, mem_max = 122270 }, + # { name = "musl", jobs = 6, cpu = 21700, mem = 57690, mem_max = 60890 }, + # { name = "aarch64", jobs = 12, cpu = 16000, mem = 27500, mem_max = 29500 }, + { name = "glibc", jobs = 2, cpu = 10000, mem = 4096, mem_max = 8192 }, + { name = "musl", jobs = 2, cpu = 10000, mem = 4096, mem_max = 8192 }, + { name = "aarch64", jobs = 2, cpu = 10000, mem = 4096, mem_max = 8192 }, ] labels = [ "buildbot-worker-${group.value.name}" ] @@ -118,9 +122,10 @@ job "buildbot-worker" { destination = "/buildroots" } + # XXX template { data = <