diff --git a/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java b/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java index 92b9b70d896..c36e907a678 100644 --- a/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java +++ b/shell/src/main/java/org/apache/accumulo/shell/commands/ScanCommand.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.Map.Entry; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.accumulo.core.classloader.ClassLoaderUtil; import org.apache.accumulo.core.client.AccumuloException; @@ -55,11 +56,16 @@ import org.apache.commons.cli.Options; import org.apache.hadoop.io.Text; +import com.google.common.base.Preconditions; + public class ScanCommand extends Command { private Option scanOptAuths, scanOptRow, scanOptColumns, disablePaginationOpt, formatterOpt, interpreterOpt, formatterInterpreterOpt, outputFileOpt, scanOptCf, scanOptCq; - + private Option scanOptBeginKeyRow, scanOptBeginKeyCf, scanOptBeginKeyCq, scanOptBeginKeyCv, + scanOptBeginKeyTs, scanOptEndKeyRow, scanOptEndKeyCf, scanOptEndKeyCq, scanOptEndKeyCv, + scanOptEndKeyTs, scanOptBeginKeyExclusive, scanOptEndKeyExclusive; + private Options scanOptsKeyRange; protected Option showFewOpt; protected Option timestampOpt; protected Option profileOpt; @@ -346,19 +352,33 @@ protected void fetchColumns(final CommandLine cl, final ScannerBase scanner, protected Range getRange(final CommandLine cl, @SuppressWarnings("deprecation") final org.apache.accumulo.core.util.interpret.ScanInterpreter formatter) throws UnsupportedEncodingException { - if ((cl.hasOption(OptUtil.START_ROW_OPT) || cl.hasOption(OptUtil.END_ROW_OPT)) - && cl.hasOption(scanOptRow.getOpt())) { - // did not see a way to make commons cli do this check... it has mutually exclusive options - // but does not support the or - throw new IllegalArgumentException("Options -" + scanOptRow.getOpt() + " AND (-" - + OptUtil.START_ROW_OPT + " OR -" + OptUtil.END_ROW_OPT + ") are mutually exclusive "); - } - - if (cl.hasOption(scanOptRow.getOpt())) { + final boolean hasStartOrEndRow = + cl.hasOption(OptUtil.START_ROW_OPT) || cl.hasOption(OptUtil.END_ROW_OPT); + final boolean hasScanOptRow = cl.hasOption(scanOptRow.getOpt()); + final boolean hasScanOptKeyRange = + Arrays.stream(cl.getOptions()).anyMatch(opt -> scanOptsKeyRange.getOptions().contains(opt)); + // did not see a way to make commons cli do this check... it has mutually exclusive options + // but does not support the or + Preconditions.checkArgument( + !(hasStartOrEndRow && hasScanOptRow) && !(hasStartOrEndRow && hasScanOptKeyRange) + && !(hasScanOptRow && hasScanOptKeyRange), + "Options (-%s) AND (-%s OR -%s) AND (%s) are mutually exclusive", scanOptRow.getOpt(), + OptUtil.START_ROW_OPT, OptUtil.END_ROW_OPT, scanOptsKeyRange.getOptions().stream() + .map(opt -> "-" + opt.getOpt()).collect(Collectors.joining(" OR "))); + + if (hasScanOptRow) { @SuppressWarnings("deprecation") var interprettedRow = formatter .interpretRow(new Text(cl.getOptionValue(scanOptRow.getOpt()).getBytes(Shell.CHARSET))); return new Range(interprettedRow); + } else if (hasScanOptKeyRange) { + var beginKey = createKey(cl, scanOptBeginKeyRow, scanOptBeginKeyCf, scanOptBeginKeyCq, + scanOptBeginKeyCv, scanOptBeginKeyTs); + var endKey = createKey(cl, scanOptEndKeyRow, scanOptEndKeyCf, scanOptEndKeyCq, + scanOptEndKeyCv, scanOptEndKeyTs); + final boolean beginInclusive = !cl.hasOption(scanOptBeginKeyExclusive.getOpt()); + final boolean endInclusive = !cl.hasOption(scanOptEndKeyExclusive.getOpt()); + return new Range(beginKey, beginInclusive, endKey, endInclusive); } else { Text startRow = OptUtil.getStartRow(cl); if (startRow != null) { @@ -378,6 +398,58 @@ protected Range getRange(final CommandLine cl, } } + private Key createKey(CommandLine cl, Option scanOptKeyRow, Option scanOptKeyCf, + Option scanOptKeyCq, Option scanOptKeyCv, Option scanOptKeyTs) { + requireAllPreceding(cl, scanOptKeyRow, scanOptKeyCf, scanOptKeyCq, scanOptKeyCv, scanOptKeyTs); + final byte[] emptyBytes = new byte[0]; + byte[] row, cf = emptyBytes, cq = emptyBytes, cv = emptyBytes; + long ts = Long.MAX_VALUE; + if (cl.hasOption(scanOptKeyRow)) { + row = cl.getOptionValue(scanOptKeyRow).getBytes(Shell.CHARSET); + if (cl.hasOption(scanOptKeyCf)) { + cf = cl.getOptionValue(scanOptKeyCf.getOpt()).getBytes(Shell.CHARSET); + if (cl.hasOption(scanOptKeyCq)) { + cq = cl.getOptionValue(scanOptKeyCq.getOpt()).getBytes(Shell.CHARSET); + if (cl.hasOption(scanOptKeyCv)) { + cv = cl.getOptionValue(scanOptKeyCv.getOpt()).getBytes(Shell.CHARSET); + if (cl.hasOption(scanOptKeyTs)) { + ts = Long.parseLong(cl.getOptionValue(scanOptKeyTs.getOpt())); + } + } + } + } + return new Key(row, cf, cq, cv, ts); + } + return null; + } + + private void requireAllPreceding(CommandLine cl, Option scanOptKeyRow, Option scanOptKeyCf, + Option scanOptKeyCq, Option scanOptKeyCv, Option scanOptKeyTs) { + if (cl.hasOption(scanOptKeyCf)) { + Preconditions.checkArgument(cl.hasOption(scanOptKeyRow), "-%s is required when using -%s", + scanOptKeyRow.getOpt(), scanOptKeyCf.getOpt()); + } + if (cl.hasOption(scanOptKeyCq)) { + Preconditions.checkArgument(cl.hasOption(scanOptKeyCf) && cl.hasOption(scanOptKeyRow), + "both -%s and -%s are required when using -%s", scanOptKeyRow.getOpt(), + scanOptKeyCf.getOpt(), scanOptKeyCq.getOpt()); + } + if (cl.hasOption(scanOptKeyCv)) { + Preconditions.checkArgument( + cl.hasOption(scanOptKeyCq) && cl.hasOption(scanOptKeyCf) && cl.hasOption(scanOptKeyRow), + "-%s -%s -%s are all required when using -%s", scanOptKeyRow.getOpt(), + scanOptKeyCf.getOpt(), scanOptKeyCq.getOpt(), scanOptKeyCv.getOpt()); + } + if (cl.hasOption(scanOptKeyTs)) { + Preconditions.checkArgument( + cl.hasOption(scanOptKeyCv) && cl.hasOption(scanOptKeyCq) && cl.hasOption(scanOptKeyCf) + && cl.hasOption(scanOptKeyRow), + "-%s -%s -%s -%s are all required when using -%s", scanOptKeyRow.getOpt(), + scanOptKeyCf.getOpt(), scanOptKeyCq.getOpt(), scanOptKeyCv.getOpt(), + scanOptKeyTs.getOpt()); + } + } + protected Authorizations getAuths(final CommandLine cl, final Shell shellState) throws AccumuloSecurityException, AccumuloException { final String user = shellState.getAccumuloClient().whoami(); @@ -409,16 +481,40 @@ public Options getOptions() { "scan authorizations (all user auths are used if this argument is not specified)"); optStartRowExclusive = new Option("be", "begin-exclusive", false, "make start row exclusive (by default it's inclusive)"); - optStartRowExclusive.setArgName("begin-exclusive"); optEndRowExclusive = new Option("ee", "end-exclusive", false, "make end row exclusive (by default it's inclusive)"); - optEndRowExclusive.setArgName("end-exclusive"); scanOptRow = new Option("r", "row", true, "row to scan"); scanOptColumns = new Option("c", "columns", true, "comma-separated columns. This option is mutually exclusive with cf and cq"); scanOptCf = new Option("cf", "column-family", true, "column family to scan."); scanOptCq = new Option("cq", "column-qualifier", true, "column qualifier to scan"); + scanOptBeginKeyRow = new Option("bkr", "begin-key-r", true, "key-based range start row"); + scanOptBeginKeyCf = + new Option("bkcf", "begin-key-cf", true, "key-based range start column family"); + scanOptBeginKeyCq = + new Option("bkcq", "begin-key-cq", true, "key-based range start column qualifier"); + scanOptBeginKeyCv = + new Option("bkcv", "begin-key-cv", true, "key-based range start column visibility"); + scanOptBeginKeyTs = new Option("bkts", "begin-key-ts", true, "key-based range start timestamp"); + scanOptEndKeyRow = new Option("ekr", "end-key-r", true, "key-based range end row"); + scanOptEndKeyCf = new Option("ekcf", "end-key-cf", true, "key-based range end column family"); + scanOptEndKeyCq = + new Option("ekcq", "end-key-cq", true, "key-based range end column qualifier"); + scanOptEndKeyCv = + new Option("ekcv", "end-key-cv", true, "key-based range end column visibility"); + scanOptEndKeyTs = new Option("ekts", "end-key-ts", true, "key-based range end timestamp"); + scanOptBeginKeyExclusive = new Option("bke", "begin-key-exclusive", false, + "make start key exclusive (by default it's inclusive)"); + scanOptEndKeyExclusive = new Option("eke", "end-key-exclusive", false, + "make end key exclusive (by default it's inclusive)"); + scanOptsKeyRange = new Options(); + scanOptsKeyRange.addOption(scanOptBeginKeyRow).addOption(scanOptBeginKeyCf) + .addOption(scanOptBeginKeyCq).addOption(scanOptBeginKeyCv).addOption(scanOptBeginKeyTs) + .addOption(scanOptEndKeyRow).addOption(scanOptEndKeyCf).addOption(scanOptEndKeyCq) + .addOption(scanOptEndKeyCv).addOption(scanOptEndKeyTs).addOption(scanOptBeginKeyExclusive) + .addOption(scanOptEndKeyExclusive); + timestampOpt = new Option("st", "show-timestamps", false, "display timestamps"); disablePaginationOpt = new Option("np", "no-pagination", false, "disable pagination of output"); showFewOpt = new Option("f", "show-few", true, "show only a specified number of characters"); @@ -439,6 +535,8 @@ public Options getOptions() { new Option("cl", "consistency-level", true, "set consistency level (experimental)"); scanOptAuths.setArgName("comma-separated-authorizations"); + optStartRowExclusive.setArgName("begin-exclusive"); + optEndRowExclusive.setArgName("end-exclusive"); scanOptRow.setArgName("row"); scanOptColumns .setArgName("[:]{,[:]}"); @@ -453,6 +551,19 @@ public Options getOptions() { executionHintsOpt.setArgName("={,=}"); scanServerOpt.setArgName("immediate|eventual"); + scanOptBeginKeyRow.setArgName("begin-key-r"); + scanOptBeginKeyCf.setArgName("begin-key-cf"); + scanOptBeginKeyCq.setArgName("begin-key-cq"); + scanOptBeginKeyCv.setArgName("begin-key-cv"); + scanOptBeginKeyTs.setArgName("begin-key-ts"); + scanOptEndKeyRow.setArgName("end-key-r"); + scanOptEndKeyCf.setArgName("end-key-cf"); + scanOptEndKeyCq.setArgName("end-key-cq"); + scanOptEndKeyCv.setArgName("end-key-cv"); + scanOptEndKeyTs.setArgName("end-key-ts"); + scanOptBeginKeyExclusive.setArgName("begin-key-exclusive"); + scanOptEndKeyExclusive.setArgName("end-key-exclusive"); + profileOpt = new Option("pn", "profile", true, "iterator profile name"); profileOpt.setArgName("profile"); @@ -487,6 +598,7 @@ public Options getOptions() { o.addOption(contextOpt); o.addOption(executionHintsOpt); o.addOption(scanServerOpt); + o.addOptions(scanOptsKeyRange); return o; } diff --git a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java index d8fc412d90e..fbe61e192e7 100644 --- a/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java +++ b/test/src/main/java/org/apache/accumulo/test/shell/ShellIT.java @@ -821,4 +821,93 @@ public void testMaxSplitsOption() throws Exception { exec("getsplits -m 0", true, "0\n1\n2\n3\n4\n5\n6\n7\n8\n9\na\nb\nc\nd\ne\nf\ng\nh\ni\nj\nk\nl\nm\nn\no\np\nq\nr\ns\nt\n"); } + + @Test + public void testScanKeyRange() throws IOException { + Shell.log.debug("Starting testScanKeyRange test ------------------"); + final String tableName = getUniqueNames(1)[0]; + exec("setauths -s vis0,vis1", true); + exec("createtable " + tableName, true); + try { + exec("config -t " + tableName + " -s table.iterator.scan.vers.opt.maxVersions=10", true); + final String scan = "scan -np -st "; + + String[] rows = new String[] {"r0", "r1"}; + String[] cfs = new String[] {"cf0", "cf1"}; + String[] cqs = new String[] {"cq0", "cq1"}; + String[] cvs = new String[] {"vis0", "vis1"}; + int[] tss = new int[] {0, 1}; + + for (String row : rows) { + for (String cf : cfs) { + for (String cq : cqs) { + for (String cv : cvs) { + for (int ts : tss) { + exec(String.format("insert %s %s %s -l %s -ts %s val", row, cf, cq, cv, ts), true); + } + } + } + } + } + + // incorrect usage + exec(scan + "-bkr r0 -r r0", false, "mutually exclusive"); + exec(scan + "-bkcf cf0 -ekcf cf1", false, "-bkr is required when using -bkcf"); + exec(scan + "-ekcf cf1", false, "-ekr is required when using -ekcf"); + exec(scan + "-bkr r0 -bkcq cq0 -ekr r0 -ekcq cq1", false, + "-bkr and -bkcf are required when using -bkcq"); + exec(scan + "-ekr r0 -ekcq cq1", false, "-ekr and -ekcf are required when using -ekcq"); + exec(scan + "-bkr r0 -bkcf cf1 -bkcv vis0 -ekr r1 -ekcf cf1 -ekcv vis0", false, + "-bkr -bkcf -bkcq are all required when using -bkcv"); + exec(scan + "-ekr r1 -ekcf cf1 -ekcv vis0", false, + "-ekr -ekcf -ekcq are all required when using -ekcv"); + exec(scan + "-bkr r0 -bkcf cf0 -bkcq cq0 -bkts 0 -ekr r1 -ekcf cf0 -ekcq cq0 -ekts 0", false, + "-bkr -bkcf -bkcq -bkcv are all required when using -bkts"); + exec(scan + "-ekr r1 -ekcf cf0 -ekcq cq0 -ekts 0", false, + "-ekr -ekcf -ekcq -ekcv are all required when using -ekts"); + + // correct usage + // range by timestamp + exec(scan + + "-bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -bkts 1 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis0 -ekts 0", + true, "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval"); + // range by visibility + exec(scan + "-bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis1", + true, "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval"); + // range by CQ + exec(scan + "-bkr r0 -bkcf cf0 -bkcq cq0 -ekr r0 -ekcf cf0 -ekcq cq1", true, + "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\n" + + "r0 cf0:cq0 [vis1] 1\tval\nr0 cf0:cq0 [vis1] 0\tval"); + // range by CF + exec(scan + "-bkr r0 -bkcf cf0 -ekr r0 -ekcf cf1", true, + "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\nr0 cf0:cq0 [vis1] 1\tval\n" + + "r0 cf0:cq0 [vis1] 0\tval\nr0 cf0:cq1 [vis0] 1\tval\nr0 cf0:cq1 [vis0] 0\tval\n" + + "r0 cf0:cq1 [vis1] 1\tval\nr0 cf0:cq1 [vis1] 0\tval"); + // range by row + exec(scan + "-bkr r0 -ekr r1", true, + "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\nr0 cf0:cq0 [vis1] 1\tval\n" + + "r0 cf0:cq0 [vis1] 0\tval\nr0 cf0:cq1 [vis0] 1\tval\nr0 cf0:cq1 [vis0] 0\tval\n" + + "r0 cf0:cq1 [vis1] 1\tval\nr0 cf0:cq1 [vis1] 0\tval\nr0 cf1:cq0 [vis0] 1\tval\n" + + "r0 cf1:cq0 [vis0] 0\tval\nr0 cf1:cq0 [vis1] 1\tval\nr0 cf1:cq0 [vis1] 0\tval\n" + + "r0 cf1:cq1 [vis0] 1\tval\nr0 cf1:cq1 [vis0] 0\tval\nr0 cf1:cq1 [vis1] 1\tval\n" + + "r0 cf1:cq1 [vis1] 0\tval"); + // only provide start options + exec(scan + "-bkr r1 -bkcf cf1 -bkcq cq1", true, + "r1 cf1:cq1 [vis0] 1\tval\nr1 cf1:cq1 [vis0] 0\tval\nr1 cf1:cq1 [vis1] 1\tval\n" + + "r1 cf1:cq1 [vis1] 0\tval"); + // only provide end options + exec(scan + "-ekr r0 -ekcf cf0 -ekcq cq1", true, + "r0 cf0:cq0 [vis0] 1\tval\nr0 cf0:cq0 [vis0] 0\tval\nr0 cf0:cq0 [vis1] 1\tval\n" + + "r0 cf0:cq0 [vis1] 0\tval"); + // test exclusivity + exec(scan + + "-bke -bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -bkts 1 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis0 -ekts 0", + true, "r0 cf0:cq0 [vis0] 0\tval"); + exec(scan + + "-eke -bkr r0 -bkcf cf0 -bkcq cq0 -bkcv vis0 -bkts 1 -ekr r0 -ekcf cf0 -ekcq cq0 -ekcv vis0 -ekts 0", + true, "r0 cf0:cq0 [vis0] 1\tval"); + } finally { + exec("deletetable " + tableName + " -f", true, "Table: [" + tableName + "] has been deleted"); + } + } }