Skip to content

Commit b8e772a

Browse files
ctruedenctrueden
authored
Merge pull request #306 from scijava/datahandles-copy
Add copy method for handles
2 parents b85a890 + fbbc76c commit b8e772a

File tree

3 files changed

+301
-32
lines changed

3 files changed

+301
-32
lines changed

‎src/main/java/org/scijava/download/DefaultDownloadService.java‎

Lines changed: 5 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@
3737

3838
import org.scijava.io.handle.DataHandle;
3939
import org.scijava.io.handle.DataHandleService;
40+
import org.scijava.io.handle.DataHandles;
4041
import org.scijava.io.location.Location;
4142
import org.scijava.plugin.Parameter;
4243
import org.scijava.plugin.Plugin;
@@ -70,7 +71,7 @@ public Download download(final Location source, final Location destination) {
7071
destination))
7172
{
7273
task.setStatusMessage("Downloading " + source.getURI());
73-
copy(task, in, out);
74+
DataHandles.copy(in, out, task);
7475
}
7576
catch (final IOException exc) {
7677
// TODO: Improve error handling:
@@ -102,13 +103,13 @@ public Download download(final Location source, final Location destination,
102103
if (isCachedHandleValid(source, cache, sourceHandle, cachedHandle)) {
103104
// The data is cached; download from the cached source instead.
104105
task.setStatusMessage("Retrieving " + source.getURI());
105-
copy(task, cachedHandle, destHandle);
106+
DataHandles.copy(cachedHandle, destHandle, task);
106107
}
107108
else {
108109
// Data is not yet cached; write to the destination _and_ the cache.
109110
task.setStatusMessage("Downloading + caching " + source.getURI());
110-
copy(task, sourceHandle, //
111-
new MultiWriteHandle(cachedHandle, destHandle));
111+
DataHandles.copy(sourceHandle, //
112+
new MultiWriteHandle(cachedHandle, destHandle), task);
112113
}
113114
}
114115
catch (final IOException exc) {
@@ -122,31 +123,6 @@ public Download download(final Location source, final Location destination,
122123

123124
// -- Helper methods --
124125

125-
private void copy(final Task task, final DataHandle<Location> in,
126-
final DataHandle<Location> out) throws IOException
127-
{
128-
long length;
129-
try {
130-
length = in.length();
131-
}
132-
catch (final IOException exc) {
133-
// Assume unknown length.
134-
length = 0;
135-
}
136-
if (length > 0) task.setProgressMaximum(length);
137-
138-
final int chunkSize = 64 * 1024; // TODO: Make size configurable.
139-
final byte[] buf = new byte[chunkSize];
140-
while (true) {
141-
if (task.isCanceled()) return;
142-
final int r = in.read(buf);
143-
if (r <= 0) break; // EOF
144-
if (task.isCanceled()) return;
145-
out.write(buf, 0, r);
146-
if (length > 0) task.setProgressValue(task.getProgressValue() + r);
147-
}
148-
}
149-
150126
private boolean isCachedHandleValid(final Location source,
151127
final LocationCache cache, final DataHandle<Location> sourceHandle,
152128
final DataHandle<Location> cachedHandle) throws IOException

‎src/main/java/org/scijava/io/handle/DataHandles.java‎

Lines changed: 144 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,13 @@
99
* %%
1010
* Redistribution and use in source and binary forms, with or without
1111
* modification, are permitted provided that the following conditions are met:
12-
*
12+
*
1313
* 1. Redistributions of source code must retain the above copyright notice,
1414
* this list of conditions and the following disclaimer.
1515
* 2. Redistributions in binary form must reproduce the above copyright notice,
1616
* this list of conditions and the following disclaimer in the documentation
1717
* and/or other materials provided with the distribution.
18-
*
18+
*
1919
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
2020
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
2121
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
@@ -38,10 +38,14 @@
3838
import java.lang.reflect.InvocationTargetException;
3939
import java.lang.reflect.Method;
4040

41+
import org.scijava.io.location.Location;
42+
import org.scijava.task.Task;
43+
4144
/**
4245
* Utility methods for working with {@link DataHandle}s.
43-
*
46+
*
4447
* @author Curtis Rueden
48+
* @author Gabriel Einsdorf
4549
*/
4650
public final class DataHandles {
4751

@@ -108,4 +112,141 @@ private static synchronized void initUTFMethod() {
108112
"No usable DataOutputStream.writeUTF(String, DataOutput)", exc);
109113
}
110114
}
115+
116+
/**
117+
* Copies all bytes from the input to the output handle. Reading and writing
118+
* start at the current positions of the handles.
119+
*
120+
* @param in the input handle
121+
* @param out the output handle
122+
* @return the number of bytes copied
123+
* @throws IOException if an I/O error occurs.
124+
*/
125+
public static long copy(final DataHandle<Location> in,
126+
final DataHandle<Location> out) throws IOException
127+
{
128+
return copy(in, out, 0l, null);
129+
}
130+
131+
/**
132+
* Copies all bytes from the input to the output handle, reporting the
133+
* progress to the provided task. Reading and writing start at the current
134+
* positions of the handles.
135+
*
136+
* @param in the input handle
137+
* @param out the output handle
138+
* @param task task to report progress to
139+
* @return the number of bytes copied
140+
* @throws IOException if an I/O error occurs.
141+
*/
142+
public static long copy(final DataHandle<Location> in,
143+
final DataHandle<Location> out, final Task task) throws IOException
144+
{
145+
return copy(in, out, 0l, task);
146+
}
147+
148+
/**
149+
* Copies up to <code>length</code> bytes from the input to the output handle.
150+
* Reading and writing start at the current positions of the handles. Stops
151+
* early if there are no more bytes available from the input handle.
152+
*
153+
* @param in the input handle
154+
* @param out the output handle
155+
* @param length maximum number of bytes to copy; will copy all bytes if set
156+
* to <code>0</code>
157+
* @return the number of bytes copied
158+
* @throws IOException if an I/O error occurs.
159+
*/
160+
public static long copy(final DataHandle<Location> in,
161+
final DataHandle<Location> out, final int length) throws IOException
162+
{
163+
return copy(in, out, length, null);
164+
}
165+
166+
/**
167+
* Copies up to <code>length</code> bytes from the input to the output handle,
168+
* reporting the progress to the provided task. Reading and writing start at
169+
* the current positions of the handles. Stops early if there are no more
170+
* bytes available from the input handle.
171+
*
172+
* @param in input handle
173+
* @param out the output handle
174+
* @param length maximum number of bytes to copy; will copy all bytes if set
175+
* to <code>0</code>
176+
* @param task a task object to use for reporting the status of the copy
177+
* operation. Can be <code>null</code> if no reporting is needed.
178+
* @return the number of bytes copied
179+
* @throws IOException if an I/O error occurs.
180+
*/
181+
public static long copy(final DataHandle<Location> in,
182+
final DataHandle<Location> out, final long length, final Task task)
183+
throws IOException
184+
{
185+
return copy(in, out, length, task, 64 * 1024);
186+
}
187+
188+
/**
189+
* Copies up to <code>length</code> bytes from the input to the output handle,
190+
* reporting the progress to the provided task. Reading and writing start at
191+
* the current positions of the handles. Stops early if there are no more
192+
* bytes available from the input handle. Uses a buffer of the provided size,
193+
* instead of using the default size.
194+
*
195+
* @param in input handle
196+
* @param out the output handle
197+
* @param length maximum number of bytes to copy, will copy all bytes if set
198+
* to <code>0</code>
199+
* @param task a task object to use for reporting the status of the copy
200+
* operation. Can be <code>null</code> if no reporting is needed.
201+
* @return the number of bytes copied
202+
* @throws IOException if an I/O error occurs.
203+
*/
204+
public static long copy(final DataHandle<Location> in,
205+
final DataHandle<Location> out, final long length, final Task task,
206+
final int bufferSize) throws IOException
207+
{
208+
209+
// get length of input
210+
final long inputlength;
211+
{
212+
long i = 0;
213+
try {
214+
i = in.length();
215+
}
216+
catch (final IOException exc) {
217+
// Assume unknown length.
218+
i = 0;
219+
}
220+
inputlength = i;
221+
}
222+
223+
if (task != null) {
224+
if (length > 0) task.setProgressMaximum(length);
225+
else if (inputlength > 0) task.setProgressMaximum(inputlength);
226+
}
227+
228+
final byte[] buffer = new byte[bufferSize];
229+
long totalRead = 0;
230+
231+
while (true) {
232+
if (task != null && task.isCanceled()) break;
233+
final int r;
234+
// ensure we do not read more than required into the buffer
235+
if (length > 0 && totalRead + bufferSize > length) {
236+
int remaining = (int) (length - totalRead);
237+
r = in.read(buffer, 0, remaining);
238+
}
239+
else {
240+
r = in.read(buffer);
241+
}
242+
if (r <= 0) break; // EOF
243+
if (task != null && task.isCanceled()) break;
244+
out.write(buffer, 0, r);
245+
totalRead += r;
246+
if (task != null) {
247+
task.setProgressValue(task.getProgressValue() + r);
248+
}
249+
}
250+
return totalRead;
251+
}
111252
}

‎src/test/java/org/scijava/io/handle/DataHandlesTest.java‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
/*-
2+
* #%L
3+
* SciJava Common shared library for SciJava software.
4+
* %%
5+
* Copyright (C) 2009 - 2017 Board of Regents of the University of
6+
* Wisconsin-Madison, Broad Institute of MIT and Harvard, Max Planck
7+
* Institute of Molecular Cell Biology and Genetics, University of
8+
* Konstanz, and KNIME GmbH.
9+
* %%
10+
* Redistribution and use in source and binary forms, with or without
11+
* modification, are permitted provided that the following conditions are met:
12+
*
13+
* 1. Redistributions of source code must retain the above copyright notice,
14+
* this list of conditions and the following disclaimer.
15+
* 2. Redistributions in binary form must reproduce the above copyright notice,
16+
* this list of conditions and the following disclaimer in the documentation
17+
* and/or other materials provided with the distribution.
18+
*
19+
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20+
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21+
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22+
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
23+
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24+
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25+
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26+
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27+
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28+
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29+
* POSSIBILITY OF SUCH DAMAGE.
30+
* #L%
31+
*/
32+
33+
package org.scijava.io.handle;
34+
35+
import static org.junit.Assert.assertEquals;
36+
37+
import java.io.IOException;
38+
39+
import org.junit.After;
40+
import org.junit.Before;
41+
import org.junit.Test;
42+
import org.scijava.Context;
43+
import org.scijava.event.EventService;
44+
import org.scijava.io.location.BytesLocation;
45+
import org.scijava.io.location.Location;
46+
import org.scijava.task.DefaultTask;
47+
import org.scijava.task.Task;
48+
import org.scijava.thread.ThreadService;
49+
import org.scijava.util.MersenneTwisterFast;
50+
51+
/**
52+
* Tests for utility methods in the {@link DataHandles} class.
53+
*
54+
* @author Gabriel Einsdorf
55+
*/
56+
public class DataHandlesTest {
57+
58+
private static final int TEST_SIZE = 2_938_740;
59+
private DataHandleService handles;
60+
private Location inFile;
61+
private BytesLocation outFile;
62+
private byte[] data;
63+
private ThreadService threadService;
64+
private EventService eventService;
65+
66+
@Before
67+
public void classSetup() {
68+
final Context ctx = new Context(DataHandleService.class,
69+
ThreadService.class, EventService.class);
70+
handles = ctx.service(DataHandleService.class);
71+
threadService = ctx.service(ThreadService.class);
72+
eventService = ctx.service(EventService.class);
73+
74+
data = randomBytes(0xbabebabe);
75+
inFile = new BytesLocation(data);
76+
outFile = new BytesLocation(TEST_SIZE);
77+
}
78+
79+
@After
80+
public void cleanup() {
81+
handles.context().dispose();
82+
}
83+
84+
@Test
85+
public void testDefaultCopy() throws IOException {
86+
try (DataHandle<Location> src = handles.create(inFile);
87+
final DataHandle<Location> dest = handles.create(outFile))
88+
{
89+
DataHandles.copy(src, dest);
90+
assertHandleEquals(data, dest);
91+
}
92+
}
93+
94+
@Test
95+
public void testCopyTask() throws IOException {
96+
try (DataHandle<Location> src = handles.create(inFile);
97+
final DataHandle<Location> dest = handles.create(outFile))
98+
{
99+
final Task t = new DefaultTask(threadService, eventService);
100+
DataHandles.copy(src, dest, t);
101+
assertEquals(t.getProgressValue(), src.length());
102+
assertHandleEquals(data, dest);
103+
}
104+
}
105+
106+
@Test
107+
public void testCopyLength() throws IOException {
108+
final int sliceSize = 50_000;
109+
try (DataHandle<Location> src = handles.create(inFile);
110+
final DataHandle<Location> dest = handles.create(outFile))
111+
{
112+
DataHandles.copy(src, dest, sliceSize);
113+
final byte[] expected = new byte[sliceSize];
114+
System.arraycopy(data, 0, expected, 0, sliceSize);
115+
assertHandleEquals(expected, dest);
116+
}
117+
}
118+
119+
@Test
120+
public void testCopyLengthTask() throws IOException {
121+
final int sliceSize = 50_000;
122+
try (DataHandle<Location> src = handles.create(inFile);
123+
final DataHandle<Location> dest = handles.create(outFile))
124+
{
125+
final Task t = new DefaultTask(threadService, eventService);
126+
DataHandles.copy(src, dest, sliceSize, t);
127+
assertEquals(t.getProgressValue(), sliceSize);
128+
129+
final byte[] expected = new byte[sliceSize];
130+
System.arraycopy(data, 0, expected, 0, sliceSize);
131+
assertHandleEquals(expected, dest);
132+
}
133+
}
134+
135+
private void assertHandleEquals(final byte[] expected,
136+
final DataHandle<Location> handle) throws IOException
137+
{
138+
handle.seek(0);
139+
for (int i = 0; i < expected.length; i++) {
140+
assertEquals(expected[i], handle.readByte());
141+
}
142+
}
143+
144+
private byte[] randomBytes(final long seed) {
145+
final MersenneTwisterFast r = new MersenneTwisterFast(seed);
146+
final byte[] ldata = new byte[TEST_SIZE];
147+
for (int i = 0; i < ldata.length; i++) {
148+
ldata[i] = r.nextByte();
149+
}
150+
return ldata;
151+
}
152+
}

0 commit comments

Comments
 (0)