TEMP

Author: classabbyamp

services/nomad/build/build-rsyncd.nomad

@@ -13,9 +13,9 @@ job "build-rsyncd" {
       }
     }
 
-    volume "glibc_hostdir" {
+    volume "root_mirror" {
       type = "host"
-      source = "glibc_hostdir"
+      source = "root_mirror"
       read_only = false
     }
 
@@ -39,21 +39,20 @@ job "build-rsyncd" {
       }
 
       volume_mount {
-        volume = "glibc_hostdir"
-        destination = "/hostdir"
+        volume = "root_mirror"
+        destination = "/mirror"
       }
 
       template {
-        data = file("xbps-clean-sigs")
-        destination = "local/xbps-clean-sigs"
+        data = file("rsync-post-xfer")
+        destination = "local/rsync-post-xfer"
         perms = "0755"
       }
 
       template {
         data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" }}
-buildsync-aarch64:{{ .aarch64_password }}
-buildsync-musl:{{ .musl_password }}
+buildsync:{{ .password }}
 {{- end }}
 EOF
         destination = "secrets/buildsync.secrets"
@@ -77,20 +76,29 @@ path = /hostdir/sources
 filter = - by_sha256/ - .* - *.part
 auth users = buildsync-*:rw
 
-[aarch64]
-path = /hostdir/binpkgs/aarch64
-auth users = buildsync-aarch64:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - *.x86_64* - x86_64*-repodata - .*
-post-xfer exec = /local/xbps-clean-sigs
-
-[musl]
-path = /hostdir/binpkgs/musl
-auth users = buildsync-musl:rw
-filter = + */ + *-repodata + otime + *.xbps - *.sig - *.sig2 - *-repodata.* - *-stagedata.* - .*
-post-xfer exec = /local/xbps-clean-sigs
+&merge /local/rsyncd.d
 EOF
         destination = "local/buildsync.conf"
       }
+
+      dynamic "template" {
+        for_each = [
+          "x86_64", "i686", "armv7l", "armv6l",
+          "x86_64-musl", "armv7l-musl", "armv6l-musl",
+          "aarch64", "aarch64-musl",
+        ]
+
+        content {
+          data = <<EOF
+[incoming-${template.value}]
+path = /incoming/${template.value}
+auth users = buildsync:rw
+filter = + */ + *.xbps - *.sig - *.sig2 - *-repodata* - .*
+post-xfer exec = /local/rsync-post-xfer
+EOF
+          destination = "local/rsyncd.d/${template.value}.conf.inc"
+        }
+      }
     }
   }
 }

services/nomad/build/buildbot-worker.nomad

@@ -10,7 +10,7 @@ job "buildbot-worker" {
       // memory_max is ~95% of capacity
       { name = "glibc",   jobs = 10, cpu = 38100, mem = 115840, mem_max = 122270 },
       { name = "musl",    jobs = 6,  cpu = 21700, mem = 57690,  mem_max = 60890  },
-      { name = "aarch64", jobs = 14,  cpu = 16000, mem = 27500,  mem_max = 29500 },
+      { name = "aarch64", jobs = 12, cpu = 16000, mem = 27500,  mem_max = 29500  },
     ]
     labels = [ "buildbot-worker-${group.value.name}" ]
 
@@ -163,13 +163,19 @@ EOF
         // the builders should use local repos
         template {
           data = <<EOF
-repository=/hostdir/binpkgs/bootstrap
-repository=/hostdir/binpkgs
-repository=/hostdir/binpkgs/nonfree
+{{ range service "root-pkgs-internal" }}
 {{ if eq "${group.value.name}" "glibc" }}
-repository=/hostdir/binpkgs/multilib/bootstrap
-repository=/hostdir/binpkgs/multilib
-repository=/hostdir/binpkgs/multilib/nonfree
+repository=http://{{ .Address }}:{{ .Port }}/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}
+repository=http://{{ .Address }}:{{ .Port }}/nonfree
+repository=http://{{ .Address }}:{{ .Port }}/multilib/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/multilib
+repository=http://{{ .Address }}:{{ .Port }}/multilib/nonfree
+{{ else }}
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}/bootstrap
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}
+repository=http://{{ .Address }}:{{ .Port }}/${group.value.name}/nonfree
+{{ end }}
 {{ end }}
 EOF
           destination = "local/xbps-repos.conf"
@@ -198,7 +204,7 @@ EOF
         template {
           data = <<EOF
 {{- with nomadVar "nomad/jobs/buildsync" -}}
-{{ .${group.value.name}_password }}
+{{ .password }}
 {{- end -}}
 EOF
           destination = "secrets/rsync/password"

services/nomad/build/buildbot.cfg

@@ -10,7 +10,7 @@ from pathlib import Path
 from twisted.internet import defer
 from twisted.python import log
 
-from buildbot.process.results import SUCCESS, SKIPPED
+from buildbot.process.results import SUCCESS
 from buildbot.plugins import util, secrets, reporters, worker, schedulers
 from buildbot.plugins import steps
 
@@ -129,13 +129,19 @@ c['schedulers'].append(schedulers.ForceScheduler(
 
 # ###### BUILDERS
 
-distdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
-bulkdir = lambda f: util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+def distdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/void-packages/{f}')
+
+def bulkdir(f):
+    return util.Interpolate(f'/workdir/%(prop:name)s/xbps-bulk/{f}')
+
 hostdir = '/hostdir'
-buildroot = lambda: util.Interpolate(f'/buildroots/%(prop:name)s/buildroot')
-builddir = lambda: util.Interpolate('builddir-%(prop:buildnumber)s')
-do_sync = lambda: util.Interpolate('%(prop:sync)s') == "True"
-hide_skipped = lambda results, _: results == SKIPPED
+
+def buildroot():
+    return util.Interpolate('/buildroots/%(prop:name)s/buildroot')
+
+def builddir():
+    return util.Interpolate('builddir-%(prop:buildnumber)s')
 
 
 factory = util.BuildFactory()
@@ -229,32 +235,21 @@ def build_packages(props):
     return cmds
 
 
-@util.renderer
-def make_prune_cmd(props):
-    return ['bash', '-c',
-            util.Interpolate(f"""
-export XBPS_TARGET_ARCH="%(prop:target)s"
-for repo in / /debug /nonfree /bootstrap; do
-    xbps-rindex -r "{hostdir}/binpkgs/$repo"
-done
-if [ "$XBPS_TARGET_ARCH" = i686 ]; then
-    for repo in /multilib /multilib/nonfree /multilib/bootstrap; do
-        XBPS_TARGET_ARCH=x86_64 xbps-rindex -r "{hostdir}/binpkgs/$repo"
-    done
-fi
-""")]
-
 @util.renderer
 def make_rsync_cmd(props):
-    return ['bash', '-c',
+    return ['bash', '-e', '-c',
             util.Interpolate("""
+echo "=> Syncing packages to the shadow repository..."
 rsync -vurk --delete-after --delay-updates \
 --filter='+ */' --filter='+ %(prop:target)s-repodata' --filter='+ *.%(prop:target)s.xbps' \
 --filter='+ otime' --filter='- .*' --filter='- *' \
 --password-file=/secrets/rsync/password /hostdir/binpkgs/ \
 {{ range nomadService 1 $allocID "build-rsyncd" -}}
-rsync://buildsync-%(prop:worker)s@{{ .Address }}:{{ .Port }}/%(prop:worker)s
-{{ end -}}""")]
+rsync://buildsync@{{ .Address }}:{{ .Port }}/incoming-%(prop:target)s
+{{ end -}}
+echo "=> Cleaning built packages..."
+find /hostdir/binpkgs -name '*.xbps' -o -name '*-repodata' -print -delete
+""")]
 
 
 factory.addStep(GitWithDiff(
@@ -350,18 +345,6 @@ factory.addStep(steps.ShellSequence(
     timeout=14400,
 ))
 
-factory.addStep(steps.ShellCommand(
-    command=make_prune_cmd,
-    name='prune_packages',
-    description='removing obsolete packages',
-    descriptionDone='removed obsolete packages',
-    haltOnFailure=True,
-    logEnviron=False,
-    usePTY=True,
-    workdir='.',
-    timeout=14400,
-))
-
 factory.addStep(steps.ShellCommand(
     command=make_rsync_cmd,
     name='sync_packages',
@@ -371,8 +354,6 @@ factory.addStep(steps.ShellCommand(
     logEnviron=False,
     usePTY=True,
     workdir='.',
-    doStepIf=do_sync(),
-    hideStepIf=hide_skipped,
     timeout=14400,
     decodeRC={
         0: SUCCESS,
@@ -393,7 +374,6 @@ for b in builders:
         'target': targetarch,
         'cross': str(hostarch != targetarch),
         'worker': b['worker'],
-        'sync': str(b['sync']),
         'bootstrap_args': b.get('bootstrap_args', '-N'),
     }
 

services/nomad/build/rsync-post-xfer

@@ -0,0 +1,47 @@
+#!/bin/sh
+
+set -e
+
+process_repo() {
+    repo="$1"
+    if [ ! -d "$RSYNC_MODULE_PATH/$repo" ] || [ ! -d "/mirror/$TGT/$repo" ]; then
+        return
+    fi
+    # add to repodata
+    find "$RSYNC_MODULE_PATH"/"$repo" -maxdepth 1 -type f -printf "/mirror/$TGT/$repo/%f\0" | \
+        xargs -0 xbps-rindex -a
+    # clean old packages from repodata
+    xbps-rindex -c "/mirror/$TGT/$repo"
+    # remove old packages
+    xbps-rindex -r "/mirror/$TGT/$repo"
+    # Remove signatures that don't have a corresponding package
+    find "/mirror/$TGT/$repo" -maxdepth 1 \( -name '*.xbps.sig' -o -name '*.xbps.sig2' \) \
+        -exec sh -c 'for x in "$@"; do [ -e "${x%.sig*}" ] || rm -- $x; done' _ {} +
+}
+
+export XBPS_TARGET_ARCH="${RSYNC_MODULE_NAME#*-}"
+
+case "$XBPS_TARGET_ARCH" in
+    aarch64*) TGT="aarch64/" ;;
+    *-musl) TGT="musl/" ;;
+    *) TGT="" ;;
+esac
+
+# copy files to repo
+# TODO: args
+rsync ... "${RSYNC_MODULE_PATH:?}"/* /mirror/"$TGT"
+
+for repo in / /bootstrap /debug /nonfree; do
+    process_repo "$repo"
+done
+
+if [ "$XBPS_TARGET_ARCH" = 'i686' ]; then
+    for repo in /multilib /multilib/bootstrap /multilib/nonfree; do
+        process_repo "$repo"
+    done
+fi
+
+# clean up incoming
+rm -r "${RSYNC_MODULE_PATH:?}"/*
+
+exit 0