Skip to content

Commit c3d93f1

Browse files
authored
Merge pull request #81 from jglick/buffering
[JENKINS-54073] Buffering for FileLogStorage
2 parents 2586696 + 9edc3f8 commit c3d93f1

File tree

6 files changed

+304
-29
lines changed

6 files changed

+304
-29
lines changed

pom.xml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
<parent>
2929
<groupId>org.jenkins-ci.plugins</groupId>
3030
<artifactId>plugin</artifactId>
31-
<version>3.21</version>
31+
<version>3.25</version>
3232
<relativePath />
3333
</parent>
3434
<groupId>org.jenkins-ci.plugins.workflow</groupId>
@@ -64,22 +64,22 @@
6464
<properties>
6565
<revision>2.31</revision>
6666
<changelist>-SNAPSHOT</changelist>
67-
<jenkins.version>2.121</jenkins.version>
67+
<jenkins.version>2.121.1</jenkins.version>
6868
<java.level>8</java.level>
6969
<no-test-jar>false</no-test-jar>
70-
<workflow-support-plugin.version>2.14</workflow-support-plugin.version>
70+
<workflow-support-plugin.version>2.21</workflow-support-plugin.version>
7171
<useBeta>true</useBeta>
7272
</properties>
7373
<dependencies>
7474
<dependency>
7575
<groupId>org.jenkins-ci.plugins.workflow</groupId>
7676
<artifactId>workflow-step-api</artifactId>
77-
<version>2.10</version>
77+
<version>2.16</version>
7878
</dependency>
7979
<dependency>
8080
<groupId>org.jenkins-ci.plugins</groupId>
8181
<artifactId>scm-api</artifactId>
82-
<version>2.0.8</version>
82+
<version>2.2.6</version>
8383
</dependency>
8484
<dependency>
8585
<groupId>org.jenkins-ci.plugins.workflow</groupId>
@@ -90,13 +90,13 @@
9090
<dependency>
9191
<groupId>org.jenkins-ci.plugins.workflow</groupId>
9292
<artifactId>workflow-job</artifactId>
93-
<version>2.11.1</version>
93+
<version>2.26</version>
9494
<scope>test</scope>
9595
</dependency>
9696
<dependency>
9797
<groupId>org.jenkins-ci.plugins.workflow</groupId>
9898
<artifactId>workflow-cps</artifactId>
99-
<version>2.33</version>
99+
<version>2.58</version>
100100
<scope>test</scope>
101101
</dependency>
102102
<dependency>
@@ -121,13 +121,13 @@
121121
<dependency>
122122
<groupId>org.jenkins-ci.plugins.workflow</groupId>
123123
<artifactId>workflow-durable-task-step</artifactId>
124-
<version>2.8</version>
124+
<version>2.22</version>
125125
<scope>test</scope>
126126
</dependency>
127127
<dependency>
128128
<groupId>org.jenkins-ci.plugins</groupId>
129129
<artifactId>script-security</artifactId>
130-
<version>1.27</version>
130+
<version>1.46</version>
131131
<scope>test</scope>
132132
</dependency>
133133
<dependency>
@@ -139,7 +139,7 @@
139139
<dependency>
140140
<groupId>org.jenkins-ci.plugins</groupId>
141141
<artifactId>structs</artifactId>
142-
<version>1.14</version>
142+
<version>1.17</version>
143143
</dependency>
144144
<dependency>
145145
<groupId>org.jenkins-ci.test</groupId>
Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
/*
2+
* The MIT License
3+
*
4+
* Copyright 2018 CloudBees, Inc.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in
14+
* all copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22+
* THE SOFTWARE.
23+
*/
24+
25+
package org.jenkinsci.plugins.workflow.log;
26+
27+
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
28+
import hudson.CloseProofOutputStream;
29+
import hudson.model.BuildListener;
30+
import hudson.remoting.RemoteOutputStream;
31+
import hudson.util.StreamTaskListener;
32+
import java.io.Closeable;
33+
import java.io.IOException;
34+
import java.io.OutputStream;
35+
import java.io.PrintStream;
36+
import org.jenkinsci.remoting.SerializableOnlyOverRemoting;
37+
38+
/**
39+
* Unlike {@link StreamTaskListener} this does not set {@code autoflush} on the reconstructed {@link PrintStream}.
40+
* It also wraps on the remote side in {@link DelayBufferedOutputStream}.
41+
*/
42+
final class BufferedBuildListener implements BuildListener, Closeable, SerializableOnlyOverRemoting {
43+
44+
private final OutputStream out;
45+
@SuppressFBWarnings(value = "SE_BAD_FIELD", justification = "using Replacement anyway, fields here are irrelevant")
46+
private final PrintStream ps;
47+
48+
BufferedBuildListener(OutputStream out) throws IOException {
49+
this.out = out;
50+
ps = new PrintStream(out, false, "UTF-8");
51+
}
52+
53+
@Override public PrintStream getLogger() {
54+
return ps;
55+
}
56+
57+
@Override public void close() throws IOException {
58+
ps.close();
59+
}
60+
61+
private Object writeReplace() {
62+
return new Replacement(this);
63+
}
64+
65+
private static final class Replacement implements SerializableOnlyOverRemoting {
66+
67+
private static final long serialVersionUID = 1;
68+
69+
private final RemoteOutputStream ros;
70+
private final DelayBufferedOutputStream.Tuning tuning = DelayBufferedOutputStream.Tuning.DEFAULT; // load defaults on master
71+
72+
Replacement(BufferedBuildListener cbl) {
73+
this.ros = new RemoteOutputStream(new CloseProofOutputStream(cbl.out));
74+
}
75+
76+
private Object readResolve() throws IOException {
77+
return new BufferedBuildListener(new DelayBufferedOutputStream(ros, tuning));
78+
}
79+
80+
}
81+
82+
}
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
* The MIT License
3+
*
4+
* Copyright 2018 CloudBees, Inc.
5+
*
6+
* Permission is hereby granted, free of charge, to any person obtaining a copy
7+
* of this software and associated documentation files (the "Software"), to deal
8+
* in the Software without restriction, including without limitation the rights
9+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
* copies of the Software, and to permit persons to whom the Software is
11+
* furnished to do so, subject to the following conditions:
12+
*
13+
* The above copyright notice and this permission notice shall be included in
14+
* all copies or substantial portions of the Software.
15+
*
16+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22+
* THE SOFTWARE.
23+
*/
24+
25+
package org.jenkinsci.plugins.workflow.log;
26+
27+
import java.io.BufferedOutputStream;
28+
import java.io.FilterOutputStream;
29+
import java.io.IOException;
30+
import java.io.OutputStream;
31+
import java.lang.ref.Reference;
32+
import java.lang.ref.WeakReference;
33+
import java.util.concurrent.TimeUnit;
34+
import java.util.logging.Level;
35+
import java.util.logging.Logger;
36+
import jenkins.util.Timer;
37+
import org.jenkinsci.remoting.SerializableOnlyOverRemoting;
38+
39+
/**
40+
* Buffered output stream which is guaranteed to deliver content after some time even if idle and the buffer does not fill up.
41+
* The automatic “flushing” does <em>not</em> flush the underlying stream, for example via {@code ProxyOutputStream.Flush}.
42+
*/
43+
final class DelayBufferedOutputStream extends BufferedOutputStream {
44+
45+
private static final Logger LOGGER = Logger.getLogger(DelayBufferedOutputStream.class.getName());
46+
47+
static final class Tuning implements SerializableOnlyOverRemoting {
48+
private Tuning() {}
49+
// nonfinal for Groovy scripting:
50+
long minRecurrencePeriod = Long.getLong(DelayBufferedOutputStream.class.getName() + ".minRecurrencePeriod", 1_000); // 1s
51+
long maxRecurrencePeriod = Long.getLong(DelayBufferedOutputStream.class.getName() + ".maxRecurrencePeriod", 10_000); // 10s
52+
float recurrencePeriodBackoff = Float.parseFloat(System.getProperty(DelayBufferedOutputStream.class.getName() + ".recurrencePeriodBackoff", "1.05"));
53+
int bufferSize = Integer.getInteger(DelayBufferedOutputStream.class.getName() + ".bufferSize", 1 << 16); // 64Kib
54+
static final Tuning DEFAULT = new Tuning();
55+
}
56+
57+
private final Tuning tuning;
58+
private long recurrencePeriod;
59+
60+
DelayBufferedOutputStream(OutputStream out) {
61+
this(out, Tuning.DEFAULT);
62+
}
63+
64+
DelayBufferedOutputStream(OutputStream out, Tuning tuning) {
65+
super(new FlushControlledOutputStream(out), tuning.bufferSize);
66+
this.tuning = tuning;
67+
recurrencePeriod = tuning.minRecurrencePeriod;
68+
reschedule();
69+
}
70+
71+
private void reschedule() {
72+
Timer.get().schedule(new Flush(this), recurrencePeriod, TimeUnit.MILLISECONDS);
73+
recurrencePeriod = Math.min((long) (recurrencePeriod * tuning.recurrencePeriodBackoff), tuning.maxRecurrencePeriod);
74+
}
75+
76+
/** We can only call {@link BufferedOutputStream#flushBuffer} via {@link #flush}, but we do not wish to flush the underlying stream, only write out the buffer. */
77+
private void flushBuffer() throws IOException {
78+
ThreadLocal<Boolean> enableFlush = ((FlushControlledOutputStream) out).enableFlush;
79+
boolean orig = enableFlush.get();
80+
enableFlush.set(false);
81+
try {
82+
flush();
83+
} finally {
84+
enableFlush.set(orig);
85+
}
86+
}
87+
88+
void flushAndReschedule() {
89+
// TODO as an optimization, avoid flushing the buffer if it was recently flushed anyway due to filling up
90+
try {
91+
flushBuffer();
92+
} catch (IOException x) {
93+
LOGGER.log(Level.FINE, null, x);
94+
}
95+
reschedule();
96+
}
97+
98+
@SuppressWarnings("FinalizeDeclaration") // not ideal, but PhantomReference is more of a hassle
99+
@Override protected void finalize() throws Throwable {
100+
super.finalize();
101+
// Odd that this is not the default behavior for BufferedOutputStream.
102+
flush();
103+
}
104+
105+
private static final class Flush implements Runnable {
106+
107+
/** Since there is no explicit close event, just keep flushing periodically until the stream is collected. */
108+
private final Reference<DelayBufferedOutputStream> osr;
109+
110+
Flush(DelayBufferedOutputStream os) {
111+
osr = new WeakReference<>(os);
112+
}
113+
114+
@Override public void run() {
115+
DelayBufferedOutputStream os = osr.get();
116+
if (os != null) {
117+
os.flushAndReschedule();
118+
}
119+
}
120+
121+
}
122+
123+
/** @see DelayBufferedOutputStream#flushBuffer */
124+
private static final class FlushControlledOutputStream extends FilterOutputStream {
125+
126+
private final ThreadLocal<Boolean> enableFlush = new ThreadLocal<Boolean>() {
127+
@Override protected Boolean initialValue() {
128+
return true;
129+
}
130+
};
131+
132+
FlushControlledOutputStream(OutputStream out) {
133+
super(out);
134+
}
135+
136+
@Override public void write(byte[] b, int off, int len) throws IOException {
137+
out.write(b, off, len); // super method writes one byte at a time!
138+
}
139+
140+
@Override public void flush() throws IOException {
141+
if (enableFlush.get()) {
142+
super.flush();
143+
}
144+
}
145+
146+
}
147+
148+
}

0 commit comments

Comments
 (0)