Skip to content

key-based range options to shell "scan" command #5959

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kevinrr888 wants to merge 2 commits into
base: 2.1
Choose a base branch
from
Open

key-based range options to shell "scan" command #5959

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
005809a
key-based range options to shell "scan" command
kevinrr888 Oct 16, 2025
9bba463
trivial test changes
kevinrr888 Oct 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 124 additions & 12 deletions shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import java.util.List;
import java.util.Map.Entry;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;

import org.apache.accumulo.core.classloader.ClassLoaderUtil;
import org.apache.accumulo.core.client.AccumuloException;
Expand Down Expand Up @@ -55,11 +56,16 @@
import org.apache.commons.cli.Options;
import org.apache.hadoop.io.Text;

import com.google.common.base.Preconditions;

public class ScanCommand extends Command {

private Option scanOptAuths, scanOptRow, scanOptColumns, disablePaginationOpt, formatterOpt,
interpreterOpt, formatterInterpreterOpt, outputFileOpt, scanOptCf, scanOptCq;

private Option scanOptBeginKeyRow, scanOptBeginKeyCf, scanOptBeginKeyCq, scanOptBeginKeyCv,
scanOptBeginKeyTs, scanOptEndKeyRow, scanOptEndKeyCf, scanOptEndKeyCq, scanOptEndKeyCv,
scanOptEndKeyTs, scanOptBeginKeyExclusive, scanOptEndKeyExclusive;
private Options scanOptsKeyRange;
protected Option showFewOpt;
protected Option timestampOpt;
protected Option profileOpt;
Expand Down Expand Up @@ -346,19 +352,33 @@ protected void fetchColumns(final CommandLine cl, final ScannerBase scanner,
protected Range getRange(final CommandLine cl,
@SuppressWarnings("deprecation") final org.apache.accumulo.core.util.interpret.ScanInterpreter formatter)
throws UnsupportedEncodingException {
if ((cl.hasOption(OptUtil.START_ROW_OPT) || cl.hasOption(OptUtil.END_ROW_OPT))
&& cl.hasOption(scanOptRow.getOpt())) {
// did not see a way to make commons cli do this check... it has mutually exclusive options
// but does not support the or
throw new IllegalArgumentException("Options -" + scanOptRow.getOpt() + " AND (-"
+ OptUtil.START_ROW_OPT + " OR -" + OptUtil.END_ROW_OPT + ") are mutually exclusive ");
}

if (cl.hasOption(scanOptRow.getOpt())) {
final boolean hasStartOrEndRow =
cl.hasOption(OptUtil.START_ROW_OPT) || cl.hasOption(OptUtil.END_ROW_OPT);
final boolean hasScanOptRow = cl.hasOption(scanOptRow.getOpt());
final boolean hasScanOptKeyRange =
Arrays.stream(cl.getOptions()).anyMatch(opt -> scanOptsKeyRange.getOptions().contains(opt));
// did not see a way to make commons cli do this check... it has mutually exclusive options
// but does not support the or
Preconditions.checkArgument(
!(hasStartOrEndRow && hasScanOptRow) && !(hasStartOrEndRow && hasScanOptKeyRange)
&& !(hasScanOptRow && hasScanOptKeyRange),
"Options (-%s) AND (-%s OR -%s) AND (%s) are mutually exclusive", scanOptRow.getOpt(),
OptUtil.START_ROW_OPT, OptUtil.END_ROW_OPT, scanOptsKeyRange.getOptions().stream()
.map(opt -> "-" + opt.getOpt()).collect(Collectors.joining(" OR ")));

if (hasScanOptRow) {
@SuppressWarnings("deprecation")
var interprettedRow = formatter
.interpretRow(new Text(cl.getOptionValue(scanOptRow.getOpt()).getBytes(Shell.CHARSET)));
return new Range(interprettedRow);
} else if (hasScanOptKeyRange) {
var beginKey = createKey(cl, scanOptBeginKeyRow, scanOptBeginKeyCf, scanOptBeginKeyCq,
scanOptBeginKeyCv, scanOptBeginKeyTs);
var endKey = createKey(cl, scanOptEndKeyRow, scanOptEndKeyCf, scanOptEndKeyCq,
scanOptEndKeyCv, scanOptEndKeyTs);
final boolean beginInclusive = !cl.hasOption(scanOptBeginKeyExclusive.getOpt());
final boolean endInclusive = !cl.hasOption(scanOptEndKeyExclusive.getOpt());
return new Range(beginKey, beginInclusive, endKey, endInclusive);
} else {
Text startRow = OptUtil.getStartRow(cl);
if (startRow != null) {
Expand All @@ -378,6 +398,58 @@ protected Range getRange(final CommandLine cl,
}
}

private Key createKey(CommandLine cl, Option scanOptKeyRow, Option scanOptKeyCf,
Option scanOptKeyCq, Option scanOptKeyCv, Option scanOptKeyTs) {
requireAllPreceding(cl, scanOptKeyRow, scanOptKeyCf, scanOptKeyCq, scanOptKeyCv, scanOptKeyTs);
final byte[] emptyBytes = new byte[0];
byte[] row, cf = emptyBytes, cq = emptyBytes, cv = emptyBytes;
long ts = Long.MAX_VALUE;
if (cl.hasOption(scanOptKeyRow)) {
row = cl.getOptionValue(scanOptKeyRow).getBytes(Shell.CHARSET);
if (cl.hasOption(scanOptKeyCf)) {
cf = cl.getOptionValue(scanOptKeyCf.getOpt()).getBytes(Shell.CHARSET);
if (cl.hasOption(scanOptKeyCq)) {
cq = cl.getOptionValue(scanOptKeyCq.getOpt()).getBytes(Shell.CHARSET);
if (cl.hasOption(scanOptKeyCv)) {
cv = cl.getOptionValue(scanOptKeyCv.getOpt()).getBytes(Shell.CHARSET);
if (cl.hasOption(scanOptKeyTs)) {
ts = Long.parseLong(cl.getOptionValue(scanOptKeyTs.getOpt()));
}
}
}
}
return new Key(row, cf, cq, cv, ts);
}
return null;
}

private void requireAllPreceding(CommandLine cl, Option scanOptKeyRow, Option scanOptKeyCf,
Option scanOptKeyCq, Option scanOptKeyCv, Option scanOptKeyTs) {
if (cl.hasOption(scanOptKeyCf)) {
Preconditions.checkArgument(cl.hasOption(scanOptKeyRow), "-%s is required when using -%s",
scanOptKeyRow.getOpt(), scanOptKeyCf.getOpt());
}
if (cl.hasOption(scanOptKeyCq)) {
Preconditions.checkArgument(cl.hasOption(scanOptKeyCf) && cl.hasOption(scanOptKeyRow),
"both -%s and -%s are required when using -%s", scanOptKeyRow.getOpt(),
scanOptKeyCf.getOpt(), scanOptKeyCq.getOpt());
}
if (cl.hasOption(scanOptKeyCv)) {
Preconditions.checkArgument(
cl.hasOption(scanOptKeyCq) && cl.hasOption(scanOptKeyCf) && cl.hasOption(scanOptKeyRow),
"-%s -%s -%s are all required when using -%s", scanOptKeyRow.getOpt(),
scanOptKeyCf.getOpt(), scanOptKeyCq.getOpt(), scanOptKeyCv.getOpt());
}
if (cl.hasOption(scanOptKeyTs)) {
Preconditions.checkArgument(
cl.hasOption(scanOptKeyCv) && cl.hasOption(scanOptKeyCq) && cl.hasOption(scanOptKeyCf)
&& cl.hasOption(scanOptKeyRow),
"-%s -%s -%s -%s are all required when using -%s", scanOptKeyRow.getOpt(),
scanOptKeyCf.getOpt(), scanOptKeyCq.getOpt(), scanOptKeyCv.getOpt(),
scanOptKeyTs.getOpt());
}
}

protected Authorizations getAuths(final CommandLine cl, final Shell shellState)
throws AccumuloSecurityException, AccumuloException {
final String user = shellState.getAccumuloClient().whoami();
Expand Down Expand Up @@ -409,16 +481,40 @@ public Options getOptions() {
"scan authorizations (all user auths are used if this argument is not specified)");
optStartRowExclusive = new Option("be", "begin-exclusive", false,
"make start row exclusive (by default it's inclusive)");
optStartRowExclusive.setArgName("begin-exclusive");
optEndRowExclusive = new Option("ee", "end-exclusive", false,
"make end row exclusive (by default it's inclusive)");
optEndRowExclusive.setArgName("end-exclusive");
scanOptRow = new Option("r", "row", true, "row to scan");
scanOptColumns = new Option("c", "columns", true,
"comma-separated columns. This option is mutually exclusive with cf and cq");
scanOptCf = new Option("cf", "column-family", true, "column family to scan.");
scanOptCq = new Option("cq", "column-qualifier", true, "column qualifier to scan");

scanOptBeginKeyRow = new Option("bkr", "begin-key-r", true, "key-based range start row");
scanOptBeginKeyCf =
new Option("bkcf", "begin-key-cf", true, "key-based range start column family");
scanOptBeginKeyCq =
new Option("bkcq", "begin-key-cq", true, "key-based range start column qualifier");
scanOptBeginKeyCv =
new Option("bkcv", "begin-key-cv", true, "key-based range start column visibility");
scanOptBeginKeyTs = new Option("bkts", "begin-key-ts", true, "key-based range start timestamp");
scanOptEndKeyRow = new Option("ekr", "end-key-r", true, "key-based range end row");
scanOptEndKeyCf = new Option("ekcf", "end-key-cf", true, "key-based range end column family");
scanOptEndKeyCq =
new Option("ekcq", "end-key-cq", true, "key-based range end column qualifier");
scanOptEndKeyCv =
new Option("ekcv", "end-key-cv", true, "key-based range end column visibility");
scanOptEndKeyTs = new Option("ekts", "end-key-ts", true, "key-based range end timestamp");
scanOptBeginKeyExclusive = new Option("bke", "begin-key-exclusive", false,
"make start key exclusive (by default it's inclusive)");
scanOptEndKeyExclusive = new Option("eke", "end-key-exclusive", false,
"make end key exclusive (by default it's inclusive)");
scanOptsKeyRange = new Options();
scanOptsKeyRange.addOption(scanOptBeginKeyRow).addOption(scanOptBeginKeyCf)
.addOption(scanOptBeginKeyCq).addOption(scanOptBeginKeyCv).addOption(scanOptBeginKeyTs)
.addOption(scanOptEndKeyRow).addOption(scanOptEndKeyCf).addOption(scanOptEndKeyCq)
.addOption(scanOptEndKeyCv).addOption(scanOptEndKeyTs).addOption(scanOptBeginKeyExclusive)
.addOption(scanOptEndKeyExclusive);

timestampOpt = new Option("st", "show-timestamps", false, "display timestamps");
disablePaginationOpt = new Option("np", "no-pagination", false, "disable pagination of output");
showFewOpt = new Option("f", "show-few", true, "show only a specified number of characters");
Expand All @@ -439,6 +535,8 @@ public Options getOptions() {
new Option("cl", "consistency-level", true, "set consistency level (experimental)");

scanOptAuths.setArgName("comma-separated-authorizations");
optStartRowExclusive.setArgName("begin-exclusive");
optEndRowExclusive.setArgName("end-exclusive");
scanOptRow.setArgName("row");
scanOptColumns
.setArgName("<columnfamily>[:<columnqualifier>]{,<columnfamily>[:<columnqualifier>]}");
Expand All @@ -453,6 +551,19 @@ public Options getOptions() {
executionHintsOpt.setArgName("<key>=<value>{,<key>=<value>}");
scanServerOpt.setArgName("immediate|eventual");

scanOptBeginKeyRow.setArgName("begin-key-r");
scanOptBeginKeyCf.setArgName("begin-key-cf");
scanOptBeginKeyCq.setArgName("begin-key-cq");
scanOptBeginKeyCv.setArgName("begin-key-cv");
scanOptBeginKeyTs.setArgName("begin-key-ts");
scanOptEndKeyRow.setArgName("end-key-r");
scanOptEndKeyCf.setArgName("end-key-cf");
scanOptEndKeyCq.setArgName("end-key-cq");
scanOptEndKeyCv.setArgName("end-key-cv");
scanOptEndKeyTs.setArgName("end-key-ts");
scanOptBeginKeyExclusive.setArgName("begin-key-exclusive");
scanOptEndKeyExclusive.setArgName("end-key-exclusive");

profileOpt = new Option("pn", "profile", true, "iterator profile name");
profileOpt.setArgName("profile");

Expand Down Expand Up @@ -487,6 +598,7 @@ public Options getOptions() {
o.addOption(contextOpt);
o.addOption(executionHintsOpt);
o.addOption(scanServerOpt);
o.addOptions(scanOptsKeyRange);

return o;
}
Expand Down
89 changes: 89 additions & 0 deletions test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -821,4 +821,93 @@ public void testMaxSplitsOption() throws Exception {
exec("getsplits -m 0", true,
"0\n1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\nq\nr\ns\nt\n");
}

@Test
public void testScanKeyRange() throws IOException {
Shell.log.debug("Starting testScanKeyRange test ------------------");
final String tableName = getUniqueNames(1)[0];
exec("setauths -s vis0,vis1", true);
exec("createtable " + tableName, true);
try {
exec("config -t " + tableName + " -s table.iterator.scan.vers.opt.maxVersions=10", true);
final String scan = "scan -np -st ";

String[] rows = new String[] {"r0", "r1"};
String[] cfs = new String[] {"cf0", "cf1"};
String[] cqs = new String[] {"cq0", "cq1"};
String[] cvs = new String[] {"vis0", "vis1"};
int[] tss = new int[] {0, 1};

for (String row : rows) {
for (String cf : cfs) {
for (String cq : cqs) {
for (String cv : cvs) {
for (int ts : tss) {
exec(String.format("insert %s %s %s -l %s -ts %s val", row, cf, cq, cv, ts), true);
}
}
}
}
}

// incorrect usage
exec(scan + "-bkr r0 -r r0", false, "mutually exclusive");
exec(scan + "-bkcf cf0 -ekcf cf1", false, "-bkr is required when using -bkcf");
exec(scan + "-ekcf cf1", false, "-ekr is required when using -ekcf");
exec(scan + "-bkr r0 -bkcq cq0 -ekr r0 -ekcq cq1", false,
"-bkr and -bkcf are required when using -bkcq");
exec(scan + "-ekr r0 -ekcq cq1", false, "-ekr and -ekcf are required when using -ekcq");
exec(scan + "-bkr r0 -bkcf cf1 -bkcv vis0 -ekr r1 -ekcf cf1 -ekcv vis0", false,
"-bkr -bkcf -bkcq are all required when using -bkcv");
exec(scan + "-ekr r1 -ekcf cf1 -ekcv vis0", false,
"-ekr -ekcf -ekcq are all required when using -ekcv");
exec(scan + "-bkr r0 -bkcf cf0 -bkcq cq0 -bkts 0 -ekr r1 -ekcf cf0 -ekcq cq0 -ekts 0", false,
"-bkr -bkcf -bkcq -bkcv are all required when using -bkts");
exec(scan + "-ekr r1 -ekcf cf0 -ekcq cq0 -ekts 0", false,
"-ekr -ekcf -ekcq -ekcv are all required when using -ekts");

// correct usage
// range by timestamp
exec(scan
+ "-bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -bkts 1 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis0 -ekts 0",
true, "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval");
// range by visibility
exec(scan + "-bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis1",
true, "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval");
// range by CQ
exec(scan + "-bkr r0 -bkcf cf0 -bkcq cq0 -ekr r0 -ekcf cf0 -ekcq cq1", true,
"r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\n"
+ "r0 cf0:cq0 [vis1] 1\tval\nr0 cf0:cq0 [vis1] 0\tval");
// range by CF
exec(scan + "-bkr r0 -bkcf cf0 -ekr r0 -ekcf cf1", true,
"r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\nr0 cf0:cq0 [vis1] 1\tval\n"
+ "r0 cf0:cq0 [vis1] 0\tval\nr0 cf0:cq1 [vis0] 1\tval\nr0 cf0:cq1 [vis0] 0\tval\n"
+ "r0 cf0:cq1 [vis1] 1\tval\nr0 cf0:cq1 [vis1] 0\tval");
// range by row
exec(scan + "-bkr r0 -ekr r1", true,
"r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\nr0 cf0:cq0 [vis1] 1\tval\n"
+ "r0 cf0:cq0 [vis1] 0\tval\nr0 cf0:cq1 [vis0] 1\tval\nr0 cf0:cq1 [vis0] 0\tval\n"
+ "r0 cf0:cq1 [vis1] 1\tval\nr0 cf0:cq1 [vis1] 0\tval\nr0 cf1:cq0 [vis0] 1\tval\n"
+ "r0 cf1:cq0 [vis0] 0\tval\nr0 cf1:cq0 [vis1] 1\tval\nr0 cf1:cq0 [vis1] 0\tval\n"
+ "r0 cf1:cq1 [vis0] 1\tval\nr0 cf1:cq1 [vis0] 0\tval\nr0 cf1:cq1 [vis1] 1\tval\n"
+ "r0 cf1:cq1 [vis1] 0\tval");
// only provide start options
exec(scan + "-bkr r1 -bkcf cf1 -bkcq cq1", true,
"r1 cf1:cq1 [vis0] 1\tval\nr1 cf1:cq1 [vis0] 0\tval\nr1 cf1:cq1 [vis1] 1\tval\n"
+ "r1 cf1:cq1 [vis1] 0\tval");
// only provide end options
exec(scan + "-ekr r0 -ekcf cf0 -ekcq cq1", true,
"r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\nr0 cf0:cq0 [vis1] 1\tval\n"
+ "r0 cf0:cq0 [vis1] 0\tval");
// test exclusivity
exec(scan
+ "-bke -bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -bkts 1 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis0 -ekts 0",
true, "r0 cf0:cq0 [vis0] 0\tval");
exec(scan
+ "-eke -bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -bkts 1 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis0 -ekts 0",
true, "r0 cf0:cq0 [vis0] 1\tval");
} finally {
exec("deletetable " + tableName + " -f", true, "Table: [" + tableName + "] has been deleted");
}
}
}