Skip to content

Commit 8ae60c3

Browse files
leodidoona-agent
andcommitted
feat(tar): add deterministic mtime for tar archives
- Add Mtime field to TarOptions struct for deterministic file timestamps - Add WithMtime() option function to set mtime - Update BuildTarCommand() to use --mtime flag when Mtime is set - Add getDeterministicMtime() helper that reuses getGitCommitTimestamp() - Update all 10 BuildTarCommand() call sites to use deterministic mtime - Add comprehensive unit tests for tar mtime functionality This ensures tar archives have deterministic file modification times based on git commit timestamps, using the same timestamp source as SBOM normalization. This improves build reproducibility by eliminating timestamp variations in tar archive metadata. The --mtime flag sets all file timestamps in the archive to the git commit time, making builds more deterministic while preserving the ability to use SOURCE_DATE_EPOCH for fully reproducible builds. Co-authored-by: Ona <no-reply@ona.com>
1 parent 3362511 commit 8ae60c3

File tree

3 files changed

+244
-0
lines changed

3 files changed

+244
-0
lines changed

pkg/leeway/build.go

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1518,6 +1518,12 @@ func (p *Package) buildYarn(buildctx *buildContext, wd, result string) (bld *pac
15181518
Commands: commands,
15191519
}
15201520

1521+
// Get deterministic mtime for tar archives
1522+
mtime, err := p.getDeterministicMtime()
1523+
if err != nil {
1524+
return nil, err
1525+
}
1526+
15211527
// let's prepare for packaging
15221528
var (
15231529
pkgCommands [][]string
@@ -1546,6 +1552,7 @@ func (p *Package) buildYarn(buildctx *buildContext, wd, result string) (bld *pac
15461552
WithOutputFile(result),
15471553
WithWorkingDir("_mirror"),
15481554
WithCompression(!buildctx.DontCompress),
1555+
WithMtime(mtime),
15491556
),
15501557
}...)
15511558
resultDir = "_mirror"
@@ -1574,13 +1581,15 @@ func (p *Package) buildYarn(buildctx *buildContext, wd, result string) (bld *pac
15741581
WithOutputFile(result),
15751582
WithWorkingDir("_pkg"),
15761583
WithCompression(!buildctx.DontCompress),
1584+
WithMtime(mtime),
15771585
),
15781586
}...)
15791587
resultDir = "_pkg"
15801588
} else if cfg.Packaging == YarnArchive {
15811589
pkgCommands = append(pkgCommands, BuildTarCommand(
15821590
WithOutputFile(result),
15831591
WithCompression(!buildctx.DontCompress),
1592+
WithMtime(mtime),
15841593
))
15851594
} else {
15861595
return nil, xerrors.Errorf("unknown Yarn packaging: %s", cfg.Packaging)
@@ -1760,11 +1769,18 @@ func (p *Package) buildGo(buildctx *buildContext, wd, result string) (res *packa
17601769
commands[PackageBuildPhaseBuild] = append(commands[PackageBuildPhaseBuild], buildCmd)
17611770
}
17621771

1772+
// Get deterministic mtime for tar archives
1773+
mtime, err := p.getDeterministicMtime()
1774+
if err != nil {
1775+
return nil, err
1776+
}
1777+
17631778
commands[PackageBuildPhasePackage] = append(commands[PackageBuildPhasePackage], []string{"rm", "-rf", "_deps"})
17641779
commands[PackageBuildPhasePackage] = append(commands[PackageBuildPhasePackage],
17651780
BuildTarCommand(
17661781
WithOutputFile(result),
17671782
WithCompression(!buildctx.DontCompress),
1783+
WithMtime(mtime),
17681784
),
17691785
)
17701786
if !cfg.DontTest && !buildctx.DontTest {
@@ -2059,6 +2075,12 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
20592075
return subjects, containerDir, nil
20602076
}
20612077

2078+
// Get deterministic mtime for tar archives
2079+
mtime, err := p.getDeterministicMtime()
2080+
if err != nil {
2081+
return nil, err
2082+
}
2083+
20622084
// Create package with improved diagnostic logging
20632085
var pkgcmds [][]string
20642086

@@ -2078,6 +2100,7 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
20782100
WithWorkingDir(containerDir),
20792101
WithSourcePaths(sourcePaths...),
20802102
WithCompression(!buildctx.DontCompress),
2103+
WithMtime(mtime),
20812104
))
20822105

20832106
commands[PackageBuildPhasePackage] = pkgcmds
@@ -2111,6 +2134,12 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
21112134
encodedMetadata := base64.StdEncoding.EncodeToString(metadataContent)
21122135
pkgCommands = append(pkgCommands, []string{"sh", "-c", fmt.Sprintf("echo %s | base64 -d > %s", encodedMetadata, dockerMetadataFile)})
21132136

2137+
// Get deterministic mtime for tar archives
2138+
mtime, err := p.getDeterministicMtime()
2139+
if err != nil {
2140+
return nil, err
2141+
}
2142+
21142143
// Prepare for packaging
21152144
sourcePaths := []string{fmt.Sprintf("./%s", dockerImageNamesFiles), fmt.Sprintf("./%s", dockerMetadataFile)}
21162145
if p.C.W.Provenance.Enabled {
@@ -2126,6 +2155,7 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
21262155
WithOutputFile(result),
21272156
WithSourcePaths(sourcePaths...),
21282157
WithCompression(!buildctx.DontCompress),
2158+
WithMtime(mtime),
21292159
)
21302160
pkgCommands = append(pkgCommands, archiveCmd)
21312161

@@ -2167,6 +2197,12 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
21672197
)
21682198
}
21692199

2200+
// Get deterministic mtime for tar archives
2201+
mtime, err := p.getDeterministicMtime()
2202+
if err != nil {
2203+
return nil, err
2204+
}
2205+
21702206
// Package everything into final tar.gz
21712207
sourcePaths := []string{"./image.tar", fmt.Sprintf("./%s", dockerImageNamesFiles), "./docker-export-metadata.json"}
21722208
if len(cfg.Metadata) > 0 {
@@ -2187,6 +2223,7 @@ func (p *Package) buildDocker(buildctx *buildContext, wd, result string) (res *p
21872223
WithOutputFile(result),
21882224
WithSourcePaths(sourcePaths...),
21892225
WithCompression(!buildctx.DontCompress),
2226+
WithMtime(mtime),
21902227
)
21912228
pkgCommands = append(pkgCommands, archiveCmd)
21922229

@@ -2374,6 +2411,18 @@ func createDockerExportMetadata(wd, version string, cfg DockerPkgConfig) error {
23742411
return nil
23752412
}
23762413

2414+
// getDeterministicMtime returns the Unix timestamp to use for tar --mtime flag.
2415+
// It uses the same timestamp source as SBOM normalization for consistency.
2416+
func (p *Package) getDeterministicMtime() (int64, error) {
2417+
timestamp, err := getGitCommitTimestamp(p.C.Git().Commit)
2418+
if err != nil {
2419+
return 0, fmt.Errorf("failed to get deterministic timestamp for tar mtime (commit: %s): %w. "+
2420+
"Ensure git is available and the repository is not a shallow clone, or set SOURCE_DATE_EPOCH environment variable",
2421+
p.C.Git().Commit, err)
2422+
}
2423+
return timestamp.Unix(), nil
2424+
}
2425+
23772426
// Update buildGeneric to use compression arg helper
23782427
func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (res *packageBuild, err error) {
23792428
cfg, ok := p.Config.(GenericPkgConfig)
@@ -2409,6 +2458,12 @@ func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (res *
24092458
}...)
24102459
}
24112460

2461+
// Get deterministic mtime for tar archives
2462+
mtime, err := p.getDeterministicMtime()
2463+
if err != nil {
2464+
return nil, err
2465+
}
2466+
24122467
// Use buildTarCommand directly which will handle compression internally
24132468
var tarCmd []string
24142469
if p.C.W.Provenance.Enabled || p.C.W.SBOM.Enabled {
@@ -2428,6 +2483,7 @@ func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (res *
24282483
WithOutputFile(result),
24292484
WithSourcePaths(sourcePaths...),
24302485
WithCompression(!buildctx.DontCompress),
2486+
WithMtime(mtime),
24312487
)
24322488
return &packageBuild{
24332489
Commands: map[PackageBuildPhase][][]string{
@@ -2450,6 +2506,7 @@ func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (res *
24502506
tarCmd = BuildTarCommand(
24512507
WithFilesFrom("/dev/null"),
24522508
WithCompression(!buildctx.DontCompress),
2509+
WithMtime(mtime),
24532510
)
24542511

24552512
return &packageBuild{
@@ -2489,13 +2546,20 @@ func (p *Package) buildGeneric(buildctx *buildContext, wd, result string) (res *
24892546
commands = append(commands, cfg.Test...)
24902547
}
24912548

2549+
// Get deterministic mtime for tar archives
2550+
mtime, err := p.getDeterministicMtime()
2551+
if err != nil {
2552+
return nil, err
2553+
}
2554+
24922555
return &packageBuild{
24932556
Commands: map[PackageBuildPhase][][]string{
24942557
PackageBuildPhaseBuild: commands,
24952558
PackageBuildPhasePackage: {
24962559
BuildTarCommand(
24972560
WithOutputFile(result),
24982561
WithCompression(!buildctx.DontCompress),
2562+
WithMtime(mtime),
24992563
),
25002564
},
25012565
},

pkg/leeway/compression.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@ type TarOptions struct {
5959

6060
// ExcludePatterns specifies patterns to exclude
6161
ExcludePatterns []string
62+
63+
// Mtime sets a deterministic modification time for all files in the archive (Unix timestamp)
64+
Mtime int64
6265
}
6366

6467
// WithOutputFile sets the output file path for the tar archive
@@ -117,6 +120,13 @@ func WithExcludePatterns(patterns ...string) func(*TarOptions) {
117120
}
118121
}
119122

123+
// WithMtime sets a deterministic modification time for all files in the archive
124+
func WithMtime(timestamp int64) func(*TarOptions) {
125+
return func(opts *TarOptions) {
126+
opts.Mtime = timestamp
127+
}
128+
}
129+
120130
// getCompressionCommand returns the appropriate compression command based on options
121131
func getCompressionCommand(algo CompressionAlgorithm, level int) string {
122132
switch algo {
@@ -185,6 +195,11 @@ func BuildTarCommand(options ...func(*TarOptions)) []string {
185195
cmd = append(cmd, "--sparse")
186196
}
187197

198+
// Add deterministic mtime if specified
199+
if opts.Mtime != 0 {
200+
cmd = append(cmd, fmt.Sprintf("--mtime=@%d", opts.Mtime))
201+
}
202+
188203
// Handle files-from case specially
189204
if opts.FilesFrom != "" {
190205
cmd = append(cmd, "--files-from", opts.FilesFrom)

pkg/leeway/compression_test.go

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
package leeway
2+
3+
import (
4+
"strings"
5+
"testing"
6+
)
7+
8+
func TestWithMtime(t *testing.T) {
9+
tests := []struct {
10+
name string
11+
mtime int64
12+
wantMtime int64
13+
}{
14+
{
15+
name: "positive timestamp",
16+
mtime: 1234567890,
17+
wantMtime: 1234567890,
18+
},
19+
{
20+
name: "zero timestamp",
21+
mtime: 0,
22+
wantMtime: 0,
23+
},
24+
{
25+
name: "recent timestamp",
26+
mtime: 1700000000,
27+
wantMtime: 1700000000,
28+
},
29+
}
30+
31+
for _, tt := range tests {
32+
t.Run(tt.name, func(t *testing.T) {
33+
opts := &TarOptions{}
34+
WithMtime(tt.mtime)(opts)
35+
36+
if opts.Mtime != tt.wantMtime {
37+
t.Errorf("WithMtime() set Mtime = %d, want %d", opts.Mtime, tt.wantMtime)
38+
}
39+
})
40+
}
41+
}
42+
43+
func TestBuildTarCommand_WithMtime(t *testing.T) {
44+
tests := []struct {
45+
name string
46+
mtime int64
47+
wantMtimeFlag bool
48+
wantFlag string
49+
}{
50+
{
51+
name: "with mtime set",
52+
mtime: 1234567890,
53+
wantMtimeFlag: true,
54+
wantFlag: "--mtime=@1234567890",
55+
},
56+
{
57+
name: "with zero mtime (not set)",
58+
mtime: 0,
59+
wantMtimeFlag: false,
60+
wantFlag: "",
61+
},
62+
{
63+
name: "with recent mtime",
64+
mtime: 1700000000,
65+
wantMtimeFlag: true,
66+
wantFlag: "--mtime=@1700000000",
67+
},
68+
}
69+
70+
for _, tt := range tests {
71+
t.Run(tt.name, func(t *testing.T) {
72+
var cmd []string
73+
if tt.mtime != 0 {
74+
cmd = BuildTarCommand(
75+
WithOutputFile("test.tar.gz"),
76+
WithMtime(tt.mtime),
77+
)
78+
} else {
79+
cmd = BuildTarCommand(
80+
WithOutputFile("test.tar.gz"),
81+
)
82+
}
83+
84+
// Check if --mtime flag is present
85+
hasMtimeFlag := false
86+
for _, arg := range cmd {
87+
if strings.HasPrefix(arg, "--mtime=") {
88+
hasMtimeFlag = true
89+
if tt.wantMtimeFlag && arg != tt.wantFlag {
90+
t.Errorf("BuildTarCommand() mtime flag = %s, want %s", arg, tt.wantFlag)
91+
}
92+
break
93+
}
94+
}
95+
96+
if tt.wantMtimeFlag && !hasMtimeFlag {
97+
t.Errorf("BuildTarCommand() missing --mtime flag, want %s", tt.wantFlag)
98+
}
99+
if !tt.wantMtimeFlag && hasMtimeFlag {
100+
t.Error("BuildTarCommand() has --mtime flag, want none")
101+
}
102+
})
103+
}
104+
}
105+
106+
func TestBuildTarCommand_MtimePosition(t *testing.T) {
107+
// Test that --mtime flag appears before -cf flag
108+
cmd := BuildTarCommand(
109+
WithOutputFile("test.tar.gz"),
110+
WithMtime(1234567890),
111+
)
112+
113+
mtimeIdx := -1
114+
cfIdx := -1
115+
116+
for i, arg := range cmd {
117+
if strings.HasPrefix(arg, "--mtime=") {
118+
mtimeIdx = i
119+
}
120+
if arg == "-cf" {
121+
cfIdx = i
122+
}
123+
}
124+
125+
if mtimeIdx == -1 {
126+
t.Error("BuildTarCommand() missing --mtime flag")
127+
}
128+
if cfIdx == -1 {
129+
t.Error("BuildTarCommand() missing -cf flag")
130+
}
131+
if mtimeIdx >= cfIdx {
132+
t.Errorf("BuildTarCommand() --mtime flag at index %d should appear before -cf at index %d", mtimeIdx, cfIdx)
133+
}
134+
}
135+
136+
func TestBuildTarCommand_MtimeWithOtherOptions(t *testing.T) {
137+
// Test that mtime works correctly with other options
138+
cmd := BuildTarCommand(
139+
WithOutputFile("test.tar.gz"),
140+
WithSourcePaths("file1.txt", "file2.txt"),
141+
WithWorkingDir("/tmp"),
142+
WithCompression(true),
143+
WithMtime(1234567890),
144+
)
145+
146+
// Verify command contains expected elements
147+
cmdStr := strings.Join(cmd, " ")
148+
149+
expectedElements := []string{
150+
"tar",
151+
"--mtime=@1234567890",
152+
"-cf",
153+
"test.tar.gz",
154+
"-C",
155+
"/tmp",
156+
"file1.txt",
157+
"file2.txt",
158+
}
159+
160+
for _, elem := range expectedElements {
161+
if !strings.Contains(cmdStr, elem) {
162+
t.Errorf("BuildTarCommand() missing expected element: %s\nFull command: %s", elem, cmdStr)
163+
}
164+
}
165+
}

0 commit comments

Comments
 (0)