diff --git a/changelog/unreleased/SOLR-12074-numeric-field-terms.yml b/changelog/unreleased/SOLR-12074-numeric-field-terms.yml new file mode 100644 index 00000000000..cc2e529a2da --- /dev/null +++ b/changelog/unreleased/SOLR-12074-numeric-field-terms.yml @@ -0,0 +1,12 @@ +# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc +title: Add new NumericField that indexes both terms and numeric Point values, to speed up term lookup and close functionality gap with Trie fields. +type: added # added, changed, fixed, deprecated, removed, dependency_update, security, other +authors: + - name: Houston Putman + nick: HoustonPutman + url: https://home.apache.org/phonebook.html?uid=houston +links: + - name: SOLR-12074 + url: https://issues.apache.org/jira/browse/SOLR-12074 + - name: SOLR-18017 + url: https://issues.apache.org/jira/browse/SOLR-18017 diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java index e7f574cd341..c9b3c3e5b20 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java @@ -147,20 +147,24 @@ public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterSta miniClusterState.client.requestWithBaseUrl(miniClusterState.nodes.get(0), reload, null); } - public QueryRequest intSetQuery(boolean dvs) { - return setQuery("numbers_i" + (dvs ? "_dv" : "")); + public QueryRequest intTrieSetQuery(boolean dvs, boolean enhancedIndex) { + return setQuery("numbers_it" + (dvs ? "_dv" : "")); } - public QueryRequest longSetQuery(boolean dvs) { - return setQuery("numbers_l" + (dvs ? "_dv" : "")); + public QueryRequest intSetQuery(boolean dvs, boolean enhancedIndex) { + return setQuery("numbers_i" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : "")); } - public QueryRequest doubleSetQuery(boolean dvs) { - return setQuery("numbers_d" + (dvs ? "_dv" : "")); + public QueryRequest longSetQuery(boolean dvs, boolean enhancedIndex) { + return setQuery("numbers_l" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : "")); } - public QueryRequest floatSetQuery(boolean dvs) { - return setQuery("numbers_f" + (dvs ? "_dv" : "")); + public QueryRequest doubleSetQuery(boolean dvs, boolean enhancedIndex) { + return setQuery("numbers_d" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : "")); + } + + public QueryRequest floatSetQuery(boolean dvs, boolean enhancedIndex) { + return setQuery("numbers_f" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : "")); } QueryRequest setQuery(String field) { @@ -175,6 +179,30 @@ QueryRequest setQuery(String field) { } } + @Benchmark + public Object intTrieSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.intTrieSetQuery(false, false).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object intTrieDvSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.intTrieSetQuery(false, false).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + @Benchmark public Object intSet( Blackhole blackhole, @@ -182,7 +210,7 @@ public Object intSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.intSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.intSetQuery(false, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -194,7 +222,7 @@ public Object longSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.longSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.longSetQuery(false, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -206,7 +234,7 @@ public Object floatSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.floatSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.floatSetQuery(false, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -218,7 +246,7 @@ public Object doubleSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.doubleSetQuery(false).process(miniClusterState.client, COLLECTION); + benchState.doubleSetQuery(false, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -230,7 +258,7 @@ public Object intDvSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.intSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.intSetQuery(true, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -242,7 +270,7 @@ public Object longDvSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.longSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.longSetQuery(true, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -254,7 +282,7 @@ public Object floatDvSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.floatSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.floatSetQuery(true, false).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } @@ -266,7 +294,103 @@ public Object doubleDvSet( MiniClusterState.MiniClusterBenchState miniClusterState) throws SolrServerException, IOException { QueryResponse response = - benchState.doubleSetQuery(true).process(miniClusterState.client, COLLECTION); + benchState.doubleSetQuery(true, false).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object intEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.intSetQuery(false, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object longEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.longSetQuery(false, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object floatEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.floatSetQuery(false, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object doubleEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.doubleSetQuery(false, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object intDvEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.intSetQuery(true, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object longDvEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.longSetQuery(true, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object floatDvEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.floatSetQuery(true, true).process(miniClusterState.client, COLLECTION); + blackhole.consume(response); + return response; + } + + @Benchmark + public Object doubleDvEnhancedSet( + Blackhole blackhole, + BenchState benchState, + MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + QueryResponse response = + benchState.doubleSetQuery(true, true).process(miniClusterState.client, COLLECTION); blackhole.consume(response); return response; } diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml index e517aea5930..df670efb9ca 100644 --- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml +++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml @@ -15,14 +15,16 @@ See the License for the specific language governing permissions and limitations under the License. --> - + + positionIncrementGap="0"/> + - @@ -45,17 +47,29 @@ + + + + + + + + + + + + id diff --git a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java index d10f810a0d5..bb00488f954 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java @@ -38,6 +38,7 @@ import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.OrdinalMap; import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Collector; @@ -63,6 +64,7 @@ import org.apache.lucene.util.CharsRefBuilder; import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.LongValues; +import org.apache.lucene.util.NumericUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.ExpandParams; import org.apache.solr.common.params.GroupParams; @@ -73,8 +75,10 @@ import org.apache.solr.core.PluginInfo; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.StrField; +import org.apache.solr.schema.TrieField; import org.apache.solr.search.CollapsingQParserPlugin; import org.apache.solr.search.DocIterator; import org.apache.solr.search.DocList; @@ -335,7 +339,8 @@ public void process(ResponseBuilder rb) throws IOException { } else { groupSet = new LongHashSet(docList.size()); NumericDocValues collapseValues = - contexts.get(currentContext).reader().getNumericDocValues(field); + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(contexts.get(currentContext).reader(), field)); for (int i = 0; i < globalDocs.length; i++) { int globalDoc = globalDocs[i]; while (globalDoc >= nextDocBase) { @@ -345,7 +350,9 @@ public void process(ResponseBuilder rb) throws IOException { currentContext + 1 < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE; - collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field); + collapseValues = + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(contexts.get(currentContext).reader(), field)); } collapsedSet.add(globalDoc); int contextDoc = globalDoc - currentDocBase; @@ -666,39 +673,75 @@ public NumericGroupExpandCollector( public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException { final int docBase = context.docBase; - final NumericDocValues docValues = context.reader().getNumericDocValues(this.field); final LeafCollector leafNullGroupCollector = expandNullGroup ? nullGroupCollector.getLeafCollector(context) : null; final LongObjectHashMap leafCollectors = new LongObjectHashMap<>(); for (LongObjectCursor entry : groups) { leafCollectors.put(entry.key, entry.value.getLeafCollector(context)); } + final SortedNumericDocValues sortedNumericDocValues = + DocValues.getSortedNumeric(context.reader(), this.field); + final NumericDocValues numericDocValues = DocValues.unwrapSingleton(sortedNumericDocValues); + if (numericDocValues != null) { + return new LeafCollector() { + + @Override + public void setScorer(Scorable scorer) throws IOException { + for (ObjectCursor c : leafCollectors.values()) { + c.value.setScorer(scorer); + } + if (expandNullGroup) { + leafNullGroupCollector.setScorer(scorer); + } + } - return new LeafCollector() { - - @Override - public void setScorer(Scorable scorer) throws IOException { - for (ObjectCursor c : leafCollectors.values()) { - c.value.setScorer(scorer); + @Override + public void collect(int docId) throws IOException { + if (numericDocValues.advanceExact(docId)) { + final long value = numericDocValues.longValue(); + final int index = leafCollectors.indexOf(value); + if (index >= 0 && !collapsedSet.contains(docId + docBase)) { + leafCollectors.indexGet(index).collect(docId); + } + } else if (expandNullGroup && !collapsedSet.contains(docId + docBase)) { + leafNullGroupCollector.collect(docId); + } } - if (expandNullGroup) { - leafNullGroupCollector.setScorer(scorer); + }; + } else { + return new LeafCollector() { + + @Override + public void setScorer(Scorable scorer) throws IOException { + for (ObjectCursor c : leafCollectors.values()) { + c.value.setScorer(scorer); + } + if (expandNullGroup) { + leafNullGroupCollector.setScorer(scorer); + } } - } - @Override - public void collect(int docId) throws IOException { - if (docValues.advanceExact(docId)) { - final long value = docValues.longValue(); - final int index = leafCollectors.indexOf(value); - if (index >= 0 && !collapsedSet.contains(docId + docBase)) { - leafCollectors.indexGet(index).collect(docId); + @Override + public void collect(int docId) throws IOException { + if (sortedNumericDocValues.advanceExact(docId)) { + long lastValue = Long.MIN_VALUE; + for (int i = 0, count = sortedNumericDocValues.docValueCount(); i < count; ++i) { + final long value = sortedNumericDocValues.nextValue(); + if (value == lastValue) { + // We don't want to collect the same value more than once + continue; + } + final int index = leafCollectors.indexOf(value); + if (index >= 0 && !collapsedSet.contains(docId + docBase)) { + leafCollectors.indexGet(index).collect(docId); + } + } + } else if (expandNullGroup && !collapsedSet.contains(docId + docBase)) { + leafNullGroupCollector.collect(docId); } - } else if (expandNullGroup && !collapsedSet.contains(docId + docBase)) { - leafNullGroupCollector.collect(docId); } - } - }; + }; + } } @Override @@ -869,9 +912,17 @@ private static String numericToString(FieldType fieldType, long val) { case LONG: return Long.toString(val); case FLOAT: - return Float.toString(Float.intBitsToFloat((int) val)); + if (fieldType instanceof PointField || fieldType instanceof TrieField) { + return Float.toString(Float.intBitsToFloat((int) val)); + } else { + return Float.toString(NumericUtils.sortableIntToFloat((int) val)); + } case DOUBLE: - return Double.toString(Double.longBitsToDouble(val)); + if (fieldType instanceof PointField || fieldType instanceof TrieField) { + return Double.toString(Double.longBitsToDouble(val)); + } else { + return Double.toString(NumericUtils.sortableLongToDouble(val)); + } case DATE: break; } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java index f77a1898771..e157ee3551e 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java @@ -23,6 +23,7 @@ import org.apache.solr.common.params.StatsParams; import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.schema.PointField; import org.apache.solr.search.DocSet; import org.apache.solr.util.SolrResponseUtil; @@ -45,9 +46,15 @@ public void prepare(ResponseBuilder rb) throws IOException { if (statsField.getSchemaField() != null && statsField.getSchemaField().getType().isPointField() && !statsField.getSchemaField().hasDocValues()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, - "Can't calculate stats on a PointField without docValues"); + if (statsField.getSchemaField().getType() instanceof PointField) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Can't calculate stats on a PointField without docValues"); + } else { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Can't calculate stats on a NumericField without docValues"); + } } } } diff --git a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java index 5a5a9b0dc97..c16d90efb60 100644 --- a/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java +++ b/solr/core/src/java/org/apache/solr/handler/component/StatsValuesFactory.java @@ -40,6 +40,7 @@ import org.apache.solr.schema.DatePointField; import org.apache.solr.schema.EnumFieldType; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.NumericField; import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.StrField; @@ -76,7 +77,9 @@ public static StatsValues createStatsValues(StatsField statsField) { return new SortedDateStatsValues(statsValues, statsField); } return statsValues; - } else if (TrieField.class.isInstance(fieldType) || PointField.class.isInstance(fieldType)) { + } else if (TrieField.class.isInstance(fieldType) + || PointField.class.isInstance(fieldType) + || NumericField.class.isInstance(fieldType)) { NumericStatsValues statsValue = new NumericStatsValues(statsField); if (sf.multiValued()) { diff --git a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java index 4ce80840064..d794446ec7f 100644 --- a/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java +++ b/solr/core/src/java/org/apache/solr/legacy/BBoxStrategy.java @@ -69,7 +69,7 @@ * #makeOverlapRatioValueSource(org.locationtech.spatial4j.shape.Rectangle, double)} works by * calculating the query bbox overlap percentage against the indexed shape overlap percentage. The * indexed shape's coordinates are retrieved from {@link - * org.apache.lucene.index.LeafReader#getNumericDocValues}. + * org.apache.lucene.index.DocValues#getNumeric}. * * @lucene.experimental */ diff --git a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java index 68a2ae59fb0..1e8cd3bc830 100644 --- a/solr/core/src/java/org/apache/solr/request/IntervalFacets.java +++ b/solr/core/src/java/org/apache/solr/request/IntervalFacets.java @@ -33,6 +33,7 @@ import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IORunnable; import org.apache.lucene.util.NumericUtils; import org.apache.solr.common.SolrException; import org.apache.solr.common.params.CommonParams; @@ -40,6 +41,7 @@ import org.apache.solr.request.IntervalFacets.FacetInterval; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.NumberType; +import org.apache.solr.schema.NumericField; import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.search.DocIterator; @@ -188,76 +190,13 @@ private int compareStart(FacetInterval o1, FacetInterval o2) { private void doCount() throws IOException { if (schemaField.getType().getNumberType() != null && (!schemaField.multiValued() || schemaField.getType().isPointField())) { - if (schemaField.multiValued()) { - getCountMultiValuedNumeric(); - } else { - getCountNumeric(); - } + getCountNumeric(); } else { getCountString(); } } private void getCountNumeric() throws IOException { - final FieldType ft = schemaField.getType(); - final String fieldName = schemaField.getName(); - final NumberType numericType = ft.getNumberType(); - if (numericType == null) { - throw new IllegalStateException(); - } - final List leaves = searcher.getIndexReader().leaves(); - - final Iterator ctxIt = leaves.iterator(); - LeafReaderContext ctx = null; - NumericDocValues longs = null; - for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) { - final int doc = docsIt.nextDoc(); - if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) { - do { - ctx = ctxIt.next(); - } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()); - assert doc >= ctx.docBase; - switch (numericType) { - case LONG: - case DATE: - case INTEGER: - longs = DocValues.getNumeric(ctx.reader(), fieldName); - break; - case FLOAT: - // TODO: this bit flipping should probably be moved to tie-break in the PQ comparator - longs = - new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) { - @Override - public long longValue() throws IOException { - return NumericUtils.sortableFloatBits((int) super.longValue()); - } - }; - break; - case DOUBLE: - // TODO: this bit flipping should probably be moved to tie-break in the PQ comparator - longs = - new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) { - @Override - public long longValue() throws IOException { - return NumericUtils.sortableDoubleBits(super.longValue()); - } - }; - break; - default: - throw new AssertionError(); - } - } - int valuesDocID = longs.docID(); - if (valuesDocID < doc - ctx.docBase) { - valuesDocID = longs.advance(doc - ctx.docBase); - } - if (valuesDocID == doc - ctx.docBase) { - accumIntervalWithValue(longs.longValue()); - } - } - } - - private void getCountMultiValuedNumeric() throws IOException { final FieldType ft = schemaField.getType(); final String fieldName = schemaField.getName(); if (ft.getNumberType() == null) { @@ -267,7 +206,8 @@ private void getCountMultiValuedNumeric() throws IOException { final Iterator ctxIt = leaves.iterator(); LeafReaderContext ctx = null; - SortedNumericDocValues longs = null; + DocIdSetIterator longs = null; + IORunnable accumulate = null; for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) { final int doc = docsIt.nextDoc(); if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) { @@ -275,14 +215,44 @@ private void getCountMultiValuedNumeric() throws IOException { ctx = ctxIt.next(); } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()); assert doc >= ctx.docBase; - longs = DocValues.getSortedNumeric(ctx.reader(), fieldName); + final SortedNumericDocValues sortedNumeric = + DocValues.getSortedNumeric(ctx.reader(), fieldName); + NumericDocValues numeric = DocValues.unwrapSingleton(sortedNumeric); + if (numeric != null) { + if (!schemaField.multiValued() && !(schemaField.getType() instanceof NumericField)) { + if (schemaField.getType().getNumberType() == NumberType.FLOAT) { + numeric = + new FilterNumericDocValues(numeric) { + @Override + public long longValue() throws IOException { + return NumericUtils.sortableFloatBits((int) super.longValue()); + } + }; + } + if (schemaField.getType().getNumberType() == NumberType.DOUBLE) { + numeric = + new FilterNumericDocValues(numeric) { + @Override + public long longValue() throws IOException { + return NumericUtils.sortableDoubleBits(super.longValue()); + } + }; + } + } + longs = numeric; + final NumericDocValues finalNumeric = numeric; + accumulate = () -> accumIntervalWithValue(finalNumeric.longValue()); + } else { + longs = sortedNumeric; + accumulate = () -> accumIntervalWithMultipleValues(sortedNumeric); + } } int valuesDocID = longs.docID(); if (valuesDocID < doc - ctx.docBase) { valuesDocID = longs.advance(doc - ctx.docBase); } if (valuesDocID == doc - ctx.docBase) { - accumIntervalWithMultipleValues(longs); + accumulate.run(); } } } @@ -294,24 +264,16 @@ private void getCountString() throws IOException { // solr docsets already exclude any deleted docs final DocIdSetIterator disi = docs.iterator(leaf); if (disi != null) { - if (schemaField.multiValued()) { - SortedSetDocValues sub = leaf.reader().getSortedSetDocValues(schemaField.getName()); - if (sub == null) { - continue; - } - final SortedDocValues singleton = DocValues.unwrapSingleton(sub); - if (singleton != null) { - // some codecs may optimize SORTED_SET storage for single-valued fields - accumIntervalsSingle(singleton, disi); - } else { - accumIntervalsMulti(sub, disi); - } + SortedSetDocValues sub = DocValues.getSortedSet(leaf.reader(), schemaField.getName()); + if (sub == null) { + continue; + } + final SortedDocValues singleton = DocValues.unwrapSingleton(sub); + if (singleton != null) { + // some codecs may optimize SORTED_SET storage for single-valued fields + accumIntervalsSingle(singleton, disi); } else { - SortedDocValues sub = leaf.reader().getSortedDocValues(schemaField.getName()); - if (sub == null) { - continue; - } - accumIntervalsSingle(sub, disi); + accumIntervalsMulti(sub, disi); } } } @@ -723,8 +685,10 @@ private BytesRef getLimitFromString(SchemaField schemaField, String value) { if ("*".equals(value)) { return null; } - if (schemaField.getType().isPointField()) { - return ((PointField) schemaField.getType()).toInternalByteRef(value); + if (schemaField.getType() instanceof PointField pointField) { + return pointField.toInternalByteRef(value); + } else if (schemaField.getType() instanceof NumericField numericField) { + return numericField.toInternalByteRef(value); } return new BytesRef(schemaField.getType().toInternal(value)); } diff --git a/solr/core/src/java/org/apache/solr/request/NumericFacets.java b/solr/core/src/java/org/apache/solr/request/NumericFacets.java index 0584a46e5f8..6e35dee3415 100644 --- a/solr/core/src/java/org/apache/solr/request/NumericFacets.java +++ b/solr/core/src/java/org/apache/solr/request/NumericFacets.java @@ -46,6 +46,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.NumberType; +import org.apache.solr.schema.NumericField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieField; import org.apache.solr.search.DocIterator; @@ -215,39 +216,26 @@ private static NamedList getCountsSingleValue( ctx = ctxIt.next(); } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()); assert doc >= ctx.docBase; - switch (numericType) { - case LONG: - case DATE: - case INTEGER: - // Long, Date and Integer - longs = DocValues.getNumeric(ctx.reader(), fieldName); - break; - case FLOAT: - // TODO: this bit flipping should probably be moved to tie-break in the PQ comparator + longs = DocValues.unwrapSingleton(DocValues.getSortedNumeric(ctx.reader(), fieldName)); + if (!(sf.getType() instanceof NumericField)) { + if (sf.getType().getNumberType() == NumberType.FLOAT) { longs = - new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) { + new FilterNumericDocValues(longs) { @Override public long longValue() throws IOException { - long bits = super.longValue(); - if (bits < 0) bits ^= 0x7fffffffffffffffL; - return bits; + return NumericUtils.sortableFloatBits((int) super.longValue()); } }; - break; - case DOUBLE: - // TODO: this bit flipping should probably be moved to tie-break in the PQ comparator + } + if (sf.getType().getNumberType() == NumberType.DOUBLE) { longs = - new FilterNumericDocValues(DocValues.getNumeric(ctx.reader(), fieldName)) { + new FilterNumericDocValues(longs) { @Override public long longValue() throws IOException { - long bits = super.longValue(); - if (bits < 0) bits ^= 0x7fffffffffffffffL; - return bits; + return NumericUtils.sortableDoubleBits(super.longValue()); } }; - break; - default: - throw new AssertionError("Unexpected type: " + numericType); + } } } int valuesDocID = longs.docID(); diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java index 04531901f44..3cb718bb17f 100644 --- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java +++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java @@ -79,6 +79,7 @@ import org.apache.solr.schema.BoolField; import org.apache.solr.schema.FieldType; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.PointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieField; import org.apache.solr.search.BitDocSet; @@ -468,8 +469,13 @@ private NamedList getTermCounts(String field, Integer mincount, ParsedP NamedList counts; SchemaField sf = searcher.getSchema().getField(field); if (sf.getType().isPointField() && !sf.hasDocValues()) { - throw new SolrException( - SolrException.ErrorCode.BAD_REQUEST, "Can't facet on a PointField without docValues"); + if (sf.getType() instanceof PointField) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Can't facet on a PointField without docValues"); + } else { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Can't facet on a NumericFIeld without docValues"); + } } FieldType ft = sf.getType(); diff --git a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java index e980e809b18..82da3fa13eb 100644 --- a/solr/core/src/java/org/apache/solr/response/DocsStreamer.java +++ b/solr/core/src/java/org/apache/solr/response/DocsStreamer.java @@ -30,13 +30,18 @@ import org.apache.solr.response.transform.DocTransformer; import org.apache.solr.schema.BinaryField; import org.apache.solr.schema.BoolField; +import org.apache.solr.schema.DateField; import org.apache.solr.schema.DatePointField; import org.apache.solr.schema.DenseVectorField; +import org.apache.solr.schema.DoubleField; import org.apache.solr.schema.DoublePointField; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.FloatField; import org.apache.solr.schema.FloatPointField; import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.IntField; import org.apache.solr.schema.IntPointField; +import org.apache.solr.schema.LongField; import org.apache.solr.schema.LongPointField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.StrField; @@ -223,6 +228,11 @@ public static Object getValue(SchemaField sf, IndexableField f) { KNOWN_TYPES.add(LongPointField.class); KNOWN_TYPES.add(DoublePointField.class); KNOWN_TYPES.add(FloatPointField.class); + KNOWN_TYPES.add(IntField.class); + KNOWN_TYPES.add(LongField.class); + KNOWN_TYPES.add(DoubleField.class); + KNOWN_TYPES.add(FloatField.class); + KNOWN_TYPES.add(DateField.class); // DenseVectorField extends FloatPointField but here we list DenseVectorField // explicitly due to KNOWN_TYPES.contains use of the KNOWN_TYPES set KNOWN_TYPES.add(DenseVectorField.class); diff --git a/solr/core/src/java/org/apache/solr/schema/DateField.java b/solr/core/src/java/org/apache/solr/schema/DateField.java new file mode 100644 index 00000000000..dac1403c77e --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/DateField.java @@ -0,0 +1,289 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.time.Instant; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValuedLongFieldSource; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; +import org.apache.solr.update.processor.TimestampUpdateProcessorFactory; +import org.apache.solr.util.DateMathParser; + +/** + * An {@code NumericField} implementation of a field for {@code Date} values with millisecond + * precision using {@code LongPoint}, {@code StringField}, {@code SortedNumericDocValuesField} and + * {@code StoredField}. + * + *

Date Format for the XML, incoming and outgoing: + * + *

+ * + * A date field shall be of the form 1995-12-31T23:59:59Z The trailing "Z" designates UTC time and + * is mandatory (See below for an explanation of UTC). Optional fractional seconds are allowed, as + * long as they do not end in a trailing 0 (but any precision beyond milliseconds will be ignored). + * All other parts are mandatory. + * + *
+ * + *

This format was derived to be standards compliant (ISO 8601) and is a more restricted form of + * the canonical + * representation of dateTime from XML schema part 2. Examples... + * + *

    + *
  • 1995-12-31T23:59:59Z + *
  • 1995-12-31T23:59:59.9Z + *
  • 1995-12-31T23:59:59.99Z + *
  • 1995-12-31T23:59:59.999Z + *
+ * + *

Note that DatePointField is lenient with regards to parsing fractional seconds + * that end in trailing zeros and will ensure that those values are indexed in the correct canonical + * format. + * + *

This FieldType also supports incoming "Date Math" strings for computing values by + * adding/rounding internals of time relative either an explicit datetime (in the format specified + * above) or the literal string "NOW", ie: "NOW+1YEAR", "NOW/DAY", + * "1995-12-31T23:59:59.999Z+5MINUTES", etc... -- see {@link DateMathParser} for more examples. + * + *

NOTE: Although it is possible to configure a DateField instance with a + * default value of "NOW" to compute a timestamp of when the document was indexed, this + * is not advisable when using SolrCloud since each replica of the document may compute a slightly + * different value. {@link TimestampUpdateProcessorFactory} is recommended instead. + * + *

Explanation of "UTC"... + * + *

+ * + * "In 1970 the Coordinated Universal Time system was devised by an international advisory group of + * technical experts within the International Telecommunication Union (ITU). The ITU felt it was + * best to designate a single abbreviation for use in all languages in order to minimize confusion. + * Since unanimous agreement could not be achieved on using either the English word order, CUT, or + * the French word order, TUC, the acronym UTC was chosen as a compromise." + * + *
+ * + * @see PointField + * @see LongPoint + */ +public class DateField extends NumericField implements DateValueFieldType { + + public DateField() { + type = NumberType.DATE; + } + + @Override + public Object toNativeType(Object val) { + if (val instanceof CharSequence) { + return DateMathParser.parseMath(null, val.toString()); + } + return super.toNativeType(val); + } + + @Override + public Query getPointFieldQuery(QParser parser, SchemaField field, String value) { + return LongPoint.newExactQuery( + field.getName(), DateMathParser.parseMath(null, value).getTime()); + } + + @Override + public Query getDocValuesFieldQuery(QParser parser, SchemaField field, String value) { + return SortedNumericDocValuesField.newSlowExactQuery( + field.getName(), DateMathParser.parseMath(null, value).getTime()); + } + + @Override + public Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + long actualMin, actualMax; + if (min == null) { + actualMin = Long.MIN_VALUE; + } else { + actualMin = DateMathParser.parseMath(null, min).getTime(); + if (!minInclusive) { + if (actualMin == Long.MAX_VALUE) return new MatchNoDocsQuery(); + ++actualMin; + } + } + if (max == null) { + actualMax = Long.MAX_VALUE; + } else { + actualMax = DateMathParser.parseMath(null, max).getTime(); + if (!maxInclusive) { + if (actualMax == Long.MIN_VALUE) return new MatchNoDocsQuery(); + --actualMax; + } + } + return LongPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + long actualMin, actualMax; + if (min == null) { + actualMin = Long.MIN_VALUE; + } else { + actualMin = DateMathParser.parseMath(null, min).getTime(); + if (!minInclusive) { + if (actualMin == Long.MAX_VALUE) return new MatchNoDocsQuery(); + ++actualMin; + } + } + if (max == null) { + actualMax = Long.MAX_VALUE; + } else { + actualMax = DateMathParser.parseMath(null, max).getTime(); + if (!maxInclusive) { + if (actualMax == Long.MIN_VALUE) return new MatchNoDocsQuery(); + --actualMax; + } + } + return SortedNumericDocValuesField.newSlowRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getPointSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] values = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = DateMathParser.parseMath(null, val).getTime(); + } + return LongPoint.newSetQuery(field.getName(), values); + } + + @Override + public Query getDocValuesSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] points = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + points[i++] = DateMathParser.parseMath(null, val).getTime(); + } + return SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return new Date(LongPoint.decodeDimension(term.bytes, term.offset)); + } + + @Override + public Object toObject(IndexableField f) { + final StoredValue storedValue = f.storedValue(); + if (storedValue != null) { + return new Date(storedValue.getLongValue()); + } + final Number val = f.numericValue(); + if (val != null) { + return new Date(val.longValue()); + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + public String storedToReadable(IndexableField f) { + return Long.toString(f.storedValue().getLongValue()); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Instant.ofEpochMilli(LongPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)) + .toString(); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + Date date = (Date) toNativeType(val.toString()); + result.grow(Long.BYTES); + result.setLength(Long.BYTES); + LongPoint.encodeDimension(date.getTime(), result.bytes(), 0); + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + return null; + } else { + return Type.LONG_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new MultiValuedLongFieldSource(field.getName(), SortedNumericSelector.Type.MIN); + } + + @Override + protected ValueSource getSingleValueSource(SortedNumericSelector.Type choice, SchemaField f) { + return new MultiValuedLongFieldSource(f.getName(), choice); + } + + @Override + public List createFields(SchemaField sf, Object value) { + Date date = + (value instanceof Date) ? ((Date) value) : DateMathParser.parseMath(null, value.toString()); + return Collections.singletonList( + new LongField.SolrLongField( + sf.getName(), + date.getTime(), + sf.indexed(), + sf.enhancedIndex(), + sf.hasDocValues(), + sf.stored())); + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + Date date = + (value instanceof Date) ? ((Date) value) : DateMathParser.parseMath(null, value.toString()); + return new LongPoint(field.getName(), date.getTime()); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), ((Date) this.toNativeType(value)).getTime()); + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/DatePointField.java b/solr/core/src/java/org/apache/solr/schema/DatePointField.java index 83e3ac91699..a1d9367a3ac 100644 --- a/solr/core/src/java/org/apache/solr/schema/DatePointField.java +++ b/solr/core/src/java/org/apache/solr/schema/DatePointField.java @@ -22,8 +22,8 @@ import java.util.Collection; import java.util.Date; import java.util.Map; -import org.apache.lucene.document.LongField; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LeafReaderContext; @@ -33,6 +33,7 @@ import org.apache.lucene.queries.function.docvalues.LongDocValues; import org.apache.lucene.queries.function.valuesource.LongFieldSource; import org.apache.lucene.queries.function.valuesource.MultiValuedLongFieldSource; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortedNumericSelector; @@ -157,28 +158,39 @@ public Object toObject(IndexableField f) { } } - @Override - protected Query getExactQuery(SchemaField field, String externalVal) { - return LongPoint.newExactQuery( - field.getName(), DateMathParser.parseMath(null, externalVal).getTime()); - } - @Override public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { assert externalVals.size() > 0; - if (!field.indexed()) { - return super.getSetQuery(parser, field, externalVals); - } - long[] values = new long[externalVals.size()]; - int i = 0; - for (String val : externalVals) { - values[i] = DateMathParser.parseMath(null, val).getTime(); - i++; + Query indexQuery = null; + long[] values = null; + if (field.indexed()) { + values = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = DateMathParser.parseMath(null, val).getTime(); + } + indexQuery = LongPoint.newSetQuery(field.getName(), values); } if (field.hasDocValues()) { - return LongField.newSetQuery(field.getName(), values); + long[] points = new long[externalVals.size()]; + if (values != null) { + points = values.clone(); + } else { + int i = 0; + for (String val : externalVals) { + points[i++] = DateMathParser.parseMath(null, val).getTime(); + } + } + Query docValuesQuery = SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + if (indexQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, docValuesQuery); + } else { + return docValuesQuery; + } + } else if (indexQuery != null) { + return indexQuery; } else { - return LongPoint.newSetQuery(field.getName(), values); + return super.getSetQuery(parser, field, externalVals); } } diff --git a/solr/core/src/java/org/apache/solr/schema/DoubleField.java b/solr/core/src/java/org/apache/solr/schema/DoubleField.java new file mode 100644 index 00000000000..93fe5628d81 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/DoubleField.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValuedDoubleFieldSource; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.NumericUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; + +/** + * An {@code NumericField} implementation of a field for {@code Double} values using {@code + * DoublePoint}, {@code StringField}, {@code SortedNumericDocValuesField} and {@code StoredField}. + * + * @see PointField + * @see DoublePoint + */ +public class DoubleField extends NumericField implements DoubleValueFieldType { + + public DoubleField() { + type = NumberType.DOUBLE; + } + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).doubleValue(); + if (val instanceof CharSequence) return Double.parseDouble(val.toString()); + return super.toNativeType(val); + } + + @Override + public Query getPointFieldQuery(QParser parser, SchemaField field, String value) { + return DoublePoint.newExactQuery(field.getName(), parseDoubleFromUser(field.getName(), value)); + } + + @Override + public Query getDocValuesFieldQuery(QParser parser, SchemaField field, String value) { + return SortedNumericDocValuesField.newSlowExactQuery( + field.getName(), + NumericUtils.doubleToSortableLong(parseDoubleFromUser(field.getName(), value))); + } + + @Override + public Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + double actualMin, actualMax; + if (min == null) { + actualMin = Double.NEGATIVE_INFINITY; + } else { + actualMin = parseDoubleFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Double.POSITIVE_INFINITY) return new MatchNoDocsQuery(); + actualMin = DoublePoint.nextUp(actualMin); + } + } + if (max == null) { + actualMax = Double.POSITIVE_INFINITY; + } else { + actualMax = parseDoubleFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Double.NEGATIVE_INFINITY) return new MatchNoDocsQuery(); + actualMax = DoublePoint.nextDown(actualMax); + } + } + return DoublePoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + double actualMin, actualMax; + if (min == null) { + actualMin = Double.NEGATIVE_INFINITY; + } else { + actualMin = parseDoubleFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Double.POSITIVE_INFINITY) return new MatchNoDocsQuery(); + actualMin = DoublePoint.nextUp(actualMin); + } + } + if (max == null) { + actualMax = Double.POSITIVE_INFINITY; + } else { + actualMax = parseDoubleFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Double.NEGATIVE_INFINITY) return new MatchNoDocsQuery(); + actualMax = DoublePoint.nextDown(actualMax); + } + } + return SortedNumericDocValuesField.newSlowRangeQuery( + field.getName(), + NumericUtils.doubleToSortableLong(actualMin), + NumericUtils.doubleToSortableLong(actualMax)); + } + + @Override + public Query getPointSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + double[] values = new double[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseDoubleFromUser(field.getName(), val); + } + return DoublePoint.newSetQuery(field.getName(), values); + } + + @Override + public Query getDocValuesSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] points = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + points[i++] = NumericUtils.doubleToSortableLong(parseDoubleFromUser(field.getName(), val)); + } + return SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return DoublePoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final StoredValue storedValue = f.storedValue(); + if (storedValue != null) { + return storedValue.getDoubleValue(); + } + final Number val = f.numericValue(); + if (val != null) { + return NumericUtils.sortableLongToDouble(val.longValue()); + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + public String storedToReadable(IndexableField f) { + return Double.toString(f.storedValue().getDoubleValue()); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Double.toString(DoublePoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Double.BYTES); + result.setLength(Double.BYTES); + DoublePoint.encodeDimension(parseDoubleFromUser(null, val.toString()), result.bytes(), 0); + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + return null; + } else { + return Type.LONG_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new MultiValuedDoubleFieldSource(field.getName(), SortedNumericSelector.Type.MIN); + } + + @Override + protected ValueSource getSingleValueSource(SortedNumericSelector.Type choice, SchemaField f) { + return new MultiValuedDoubleFieldSource(f.getName(), choice); + } + + @Override + public List createFields(SchemaField sf, Object value) { + double doubleValue = + (value instanceof Number) + ? ((Number) value).doubleValue() + : Double.parseDouble(value.toString()); + return Collections.singletonList( + new SolrDoubleField( + sf.getName(), + doubleValue, + sf.indexed(), + sf.enhancedIndex(), + sf.hasDocValues(), + sf.stored())); + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + double doubleValue = + (value instanceof Number) + ? ((Number) value).doubleValue() + : Double.parseDouble(value.toString()); + return new DoublePoint(field.getName(), doubleValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Double) this.toNativeType(value)); + } + + @Override + public Query getSpecializedExistenceQuery(QParser parser, SchemaField field) { + return DoublePoint.newRangeQuery(field.getName(), Double.NEGATIVE_INFINITY, Double.NaN); + } + + /** + * Wrapper for {@link Double#parseDouble(String)} that throws a BAD_REQUEST error if the input is + * not valid + * + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static double parseDoubleFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException( + "Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Double.parseDouble(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + static final class SolrDoubleField extends Field { + + static org.apache.lucene.document.FieldType getType( + boolean rangeIndex, boolean termIndex, boolean docValues, boolean stored) { + org.apache.lucene.document.FieldType type = new org.apache.lucene.document.FieldType(); + if (rangeIndex) { + type.setDimensions(1, Double.BYTES); + } + if (termIndex) { + type.setIndexOptions(IndexOptions.DOCS); + } + if (docValues) { + type.setDocValuesType(DocValuesType.SORTED_NUMERIC); + } + type.setTokenized(false); + type.setStored(stored); + type.freeze(); + return type; + } + + private final StoredValue storedValue; + + /** + * Creates a new DoubleField, indexing the provided point and term, storing it as a DocValue, + * and optionally storing it as a stored field. + * + * @param name field name + * @param value the double value + * @throws IllegalArgumentException if the field name or value is null. + */ + public SolrDoubleField( + String name, + double value, + boolean rangeIndex, + boolean termIndex, + boolean docValues, + boolean stored) { + super(name, getType(rangeIndex, termIndex, docValues, stored)); + fieldsData = NumericUtils.doubleToSortableLong(value); + if (stored) { + storedValue = new StoredValue(value); + } else { + storedValue = null; + } + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } + + @Override + public BytesRef binaryValue() { + byte[] encodedPoint = new byte[Double.BYTES]; + double value = getValueAsDouble(); + DoublePoint.encodeDimension(value, encodedPoint, 0); + return new BytesRef(encodedPoint); + } + + private double getValueAsDouble() { + return NumericUtils.sortableLongToDouble(numericValue().longValue()); + } + + @Override + public StoredValue storedValue() { + return storedValue; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " <" + name + ':' + getValueAsDouble() + '>'; + } + + @Override + public void setDoubleValue(double value) { + super.setLongValue(NumericUtils.doubleToSortableLong(value)); + if (storedValue != null) { + storedValue.setDoubleValue(value); + } + } + + @Override + public void setLongValue(long value) { + throw new IllegalArgumentException("cannot change value type from Double to Long"); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java index 70cc45bfd5a..2d94f0164fd 100644 --- a/solr/core/src/java/org/apache/solr/schema/DoublePointField.java +++ b/solr/core/src/java/org/apache/solr/schema/DoublePointField.java @@ -18,14 +18,15 @@ package org.apache.solr.schema; import java.util.Collection; -import org.apache.lucene.document.DoubleField; import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; import org.apache.lucene.queries.function.valuesource.MultiValuedDoubleFieldSource; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortedNumericSelector; @@ -109,27 +110,53 @@ public Object toObject(IndexableField f) { } @Override - protected Query getExactQuery(SchemaField field, String externalVal) { - return DoublePoint.newExactQuery( - field.getName(), parseDoubleFromUser(field.getName(), externalVal)); - } - - @Override - public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { - assert externalVal.size() > 0; - if (!field.indexed()) { - return super.getSetQuery(parser, field, externalVal); - } - double[] values = new double[externalVal.size()]; - int i = 0; - for (String val : externalVal) { - values[i] = parseDoubleFromUser(field.getName(), val); - i++; + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { + assert externalVals.size() > 0; + Query indexQuery = null; + double[] values = null; + if (field.indexed()) { + values = new double[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseDoubleFromUser(field.getName(), val); + } + indexQuery = DoublePoint.newSetQuery(field.getName(), values); } if (field.hasDocValues()) { - return DoubleField.newSetQuery(field.getName(), values); + long[] points = new long[externalVals.size()]; + if (values != null) { + if (field.multiValued()) { + for (int i = 0; i < values.length; i++) { + points[i] = NumericUtils.doubleToSortableLong(values[i]); + } + } else { + for (int i = 0; i < values.length; i++) { + points[i] = Double.doubleToLongBits(values[i]); + } + } + } else { + int i = 0; + if (field.multiValued()) { + for (String val : externalVals) { + points[i++] = + NumericUtils.doubleToSortableLong(parseDoubleFromUser(field.getName(), val)); + } + } else { + for (String val : externalVals) { + points[i++] = Double.doubleToLongBits(parseDoubleFromUser(field.getName(), val)); + } + } + } + Query docValuesQuery = SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + if (indexQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, docValuesQuery); + } else { + return docValuesQuery; + } + } else if (indexQuery != null) { + return indexQuery; } else { - return DoublePoint.newSetQuery(field.getName(), values); + return super.getSetQuery(parser, field, externalVals); } } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java index 91f3caa38e6..f1693e4beba 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldProperties.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldProperties.java @@ -53,6 +53,8 @@ public abstract class FieldProperties { protected static final int LARGE_FIELD = 0b1000000000000000000; protected static final int UNINVERTIBLE = 0b10000000000000000000; + protected static final int ENHANCED_INDEX = 0b100000000000000000000; + static final String[] propertyNames = { "indexed", "tokenized", @@ -73,7 +75,8 @@ public abstract class FieldProperties { "termPayloads", "useDocValuesAsStored", "large", - "uninvertible" + "uninvertible", + "enhancedIndex" }; static final Map propertyMap = new HashMap<>(); diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 2f922473e50..8298af895f0 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -397,6 +397,11 @@ public Object toObject(SchemaField sf, BytesRef term) { return toObject(f); } + /** Return whether the given field can use Term queries */ + protected boolean hasIndexedTerms(SchemaField field) { + return field.indexed(); + } + /** Given an indexed term, return the human readable representation */ public String indexedToReadable(String indexedForm) { return indexedForm; @@ -547,10 +552,14 @@ public boolean incrementToken() throws IOException { return false; } - if (isPointField()) { + if (FieldType.this instanceof PointField) { BytesRef b = ((PointField) FieldType.this).toInternalByteRef(new String(cbuf, 0, n)); bytesAtt.setBytesRef(b); + } else if (FieldType.this instanceof NumericField) { + BytesRef b = + ((NumericField) FieldType.this).toInternalByteRef(new String(cbuf, 0, n)); + bytesAtt.setBytesRef(b); } else { String s = toInternal(new String(cbuf, 0, n)); termAtt.setEmpty().append(s); @@ -857,7 +866,7 @@ protected SortField getStringSort(SchemaField field, boolean reverse) { * @see #getSortField */ protected SortField getNumericSort(SchemaField field, NumberType type, boolean reverse) { - if (field.multiValued()) { + if (field.multiValued() || this instanceof NumericField) { MultiValueSelector selector = field.type.getDefaultMultiValueSelectorForSort(field, reverse); if (null != selector) { return getSortedNumericSortField( @@ -1066,7 +1075,7 @@ protected Query getSpecializedExistenceQuery(QParser parser, SchemaField field) * {@link org.apache.lucene.search.TermQuery} but overriding queries may not */ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) { - if (field.hasDocValues() && !field.indexed()) { + if (field.hasDocValues() && !hasIndexedTerms(field)) { // match-only return getRangeQuery(parser, field, externalVal, externalVal, true, true); } else { @@ -1093,8 +1102,8 @@ public Query getFieldTermQuery(QParser parser, SchemaField field, String externa * @lucene.experimental */ public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { - if (!field.indexed()) { - // TODO: if the field isn't indexed, this feels like the wrong query type to use? + if (!hasIndexedTerms(field)) { + // TODO: if the field doesn't have terms indexed, this feels like the wrong query type to use? BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (String externalVal : externalVals) { Query subq = getFieldQuery(parser, field, externalVal); @@ -1120,7 +1129,7 @@ public Query getSetQuery(QParser parser, SchemaField field, Collection e * @return A suitable rewrite method for rewriting multi-term queries to primitive queries. */ public MultiTermQuery.RewriteMethod getRewriteMethod(QParser parser, SchemaField field) { - if (!field.indexed() && field.hasDocValues()) { + if (!hasIndexedTerms(field) && field.hasDocValues()) { return new DocValuesRewriteMethod(); } else { return MultiTermQuery.CONSTANT_SCORE_REWRITE; diff --git a/solr/core/src/java/org/apache/solr/schema/FloatField.java b/solr/core/src/java/org/apache/solr/schema/FloatField.java new file mode 100644 index 00000000000..4c002bdfafa --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/FloatField.java @@ -0,0 +1,359 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValuedFloatFieldSource; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.NumericUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; + +/** + * An {@code NumericField} implementation of a field for {@code Float} values using {@code + * FloatPoint}, {@code StringField}, {@code SortedNumericDocValuesField} and {@code StoredField}. + * + * @see PointField + * @see FloatPoint + */ +public class FloatField extends NumericField implements FloatValueFieldType { + + public FloatField() { + type = NumberType.FLOAT; + } + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).floatValue(); + if (val instanceof CharSequence) return Float.parseFloat(val.toString()); + return super.toNativeType(val); + } + + @Override + public Query getPointFieldQuery(QParser parser, SchemaField field, String value) { + return FloatPoint.newExactQuery(field.getName(), parseFloatFromUser(field.getName(), value)); + } + + @Override + public Query getDocValuesFieldQuery(QParser parser, SchemaField field, String value) { + return SortedNumericDocValuesField.newSlowExactQuery( + field.getName(), + NumericUtils.floatToSortableInt(parseFloatFromUser(field.getName(), value))); + } + + @Override + public Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + float actualMin, actualMax; + if (min == null) { + actualMin = Float.NEGATIVE_INFINITY; + } else { + actualMin = parseFloatFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery(); + actualMin = FloatPoint.nextUp(actualMin); + } + } + if (max == null) { + actualMax = Float.POSITIVE_INFINITY; + } else { + actualMax = parseFloatFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery(); + actualMax = FloatPoint.nextDown(actualMax); + } + } + return FloatPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + float actualMin, actualMax; + if (min == null) { + actualMin = Float.NEGATIVE_INFINITY; + } else { + actualMin = parseFloatFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery(); + actualMin = FloatPoint.nextUp(actualMin); + } + } + if (max == null) { + actualMax = Float.POSITIVE_INFINITY; + } else { + actualMax = parseFloatFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery(); + actualMax = FloatPoint.nextDown(actualMax); + } + } + return SortedNumericDocValuesField.newSlowRangeQuery( + field.getName(), + NumericUtils.floatToSortableInt(actualMin), + NumericUtils.floatToSortableInt(actualMax)); + } + + @Override + public Query getPointSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + float[] values = new float[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseFloatFromUser(field.getName(), val); + } + return FloatPoint.newSetQuery(field.getName(), values); + } + + @Override + public Query getDocValuesSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] points = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + points[i++] = NumericUtils.floatToSortableInt(parseFloatFromUser(field.getName(), val)); + } + return SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return FloatPoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final StoredValue storedValue = f.storedValue(); + if (storedValue != null) { + return storedValue.getFloatValue(); + } + final Number val = f.numericValue(); + if (val != null) { + return NumericUtils.sortableIntToFloat(val.intValue()); + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + public String storedToReadable(IndexableField f) { + return Float.toString(f.storedValue().getFloatValue()); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Float.toString(FloatPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Float.BYTES); + result.setLength(Float.BYTES); + FloatPoint.encodeDimension(parseFloatFromUser(null, val.toString()), result.bytes(), 0); + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + return null; + } else { + return Type.INTEGER_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new MultiValuedFloatFieldSource(field.getName(), SortedNumericSelector.Type.MIN); + } + + @Override + protected ValueSource getSingleValueSource(SortedNumericSelector.Type choice, SchemaField f) { + return new MultiValuedFloatFieldSource(f.getName(), choice); + } + + @Override + public List createFields(SchemaField sf, Object value) { + float floatValue = + (value instanceof Number) + ? ((Number) value).floatValue() + : Float.parseFloat(value.toString()); + return Collections.singletonList( + new SolrFloatField( + sf.getName(), + floatValue, + sf.indexed(), + sf.enhancedIndex(), + sf.hasDocValues(), + sf.stored())); + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + float floatValue = + (value instanceof Number) + ? ((Number) value).floatValue() + : Float.parseFloat(value.toString()); + return new FloatPoint(field.getName(), floatValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Float) this.toNativeType(value)); + } + + @Override + public Query getSpecializedExistenceQuery(QParser parser, SchemaField field) { + return FloatPoint.newRangeQuery(field.getName(), Float.NEGATIVE_INFINITY, Float.NaN); + } + + /** + * Wrapper for {@link Float#parseFloat(String)} that throws a BAD_REQUEST error if the input is + * not valid + * + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static float parseFloatFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException( + "Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Float.parseFloat(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + static final class SolrFloatField extends Field { + + static org.apache.lucene.document.FieldType getType( + boolean rangeIndex, boolean termIndex, boolean docValues, boolean stored) { + org.apache.lucene.document.FieldType type = new org.apache.lucene.document.FieldType(); + if (rangeIndex) { + type.setDimensions(1, Float.BYTES); + } + if (termIndex) { + type.setIndexOptions(IndexOptions.DOCS); + } + if (docValues) { + type.setDocValuesType(DocValuesType.SORTED_NUMERIC); + } + type.setTokenized(false); + type.setStored(stored); + type.freeze(); + return type; + } + + private final StoredValue storedValue; + + /** + * Creates a new FloatField, indexing the provided point and term, storing it as a DocValue, and + * optionally storing it as a stored field. + * + * @param name field name + * @param value the float value + * @throws IllegalArgumentException if the field name or value is null. + */ + public SolrFloatField( + String name, + float value, + boolean rangeIndex, + boolean termIndex, + boolean docValues, + boolean stored) { + super(name, getType(rangeIndex, termIndex, docValues, stored)); + fieldsData = (long) NumericUtils.floatToSortableInt(value); + if (stored) { + storedValue = new StoredValue(value); + } else { + storedValue = null; + } + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } + + @Override + public BytesRef binaryValue() { + byte[] encodedPoint = new byte[Float.BYTES]; + float value = getValueAsFloat(); + FloatPoint.encodeDimension(value, encodedPoint, 0); + return new BytesRef(encodedPoint); + } + + private float getValueAsFloat() { + return NumericUtils.sortableIntToFloat(numericValue().intValue()); + } + + @Override + public StoredValue storedValue() { + return storedValue; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " <" + name + ':' + getValueAsFloat() + '>'; + } + + @Override + public void setFloatValue(float value) { + super.setLongValue(NumericUtils.floatToSortableInt(value)); + if (storedValue != null) { + storedValue.setFloatValue(value); + } + } + + @Override + public void setLongValue(long value) { + throw new IllegalArgumentException("cannot change value type from Float to Long"); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java index 4ed65fe45c3..62e80ccd014 100644 --- a/solr/core/src/java/org/apache/solr/schema/FloatPointField.java +++ b/solr/core/src/java/org/apache/solr/schema/FloatPointField.java @@ -18,14 +18,15 @@ package org.apache.solr.schema; import java.util.Collection; -import org.apache.lucene.document.FloatField; import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.NumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; import org.apache.lucene.queries.function.valuesource.MultiValuedFloatFieldSource; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortedNumericSelector; @@ -109,27 +110,52 @@ public Object toObject(IndexableField f) { } @Override - protected Query getExactQuery(SchemaField field, String externalVal) { - return FloatPoint.newExactQuery( - field.getName(), parseFloatFromUser(field.getName(), externalVal)); - } - - @Override - public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { - assert externalVal.size() > 0; - if (!field.indexed()) { - return super.getSetQuery(parser, field, externalVal); - } - float[] values = new float[externalVal.size()]; - int i = 0; - for (String val : externalVal) { - values[i] = parseFloatFromUser(field.getName(), val); - i++; + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { + assert externalVals.size() > 0; + Query indexQuery = null; + float[] values = null; + if (field.indexed()) { + values = new float[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseFloatFromUser(field.getName(), val); + } + indexQuery = FloatPoint.newSetQuery(field.getName(), values); } if (field.hasDocValues()) { - return FloatField.newSetQuery(field.getName(), values); + long[] points = new long[externalVals.size()]; + if (values != null) { + if (field.multiValued()) { + for (int i = 0; i < values.length; i++) { + points[i] = NumericUtils.floatToSortableInt(values[i]); + } + } else { + for (int i = 0; i < values.length; i++) { + points[i] = Float.floatToIntBits(values[i]); + } + } + } else { + int i = 0; + if (field.multiValued()) { + for (String val : externalVals) { + points[i++] = NumericUtils.floatToSortableInt(parseFloatFromUser(field.getName(), val)); + } + } else { + for (String val : externalVals) { + points[i++] = Float.floatToIntBits(parseFloatFromUser(field.getName(), val)); + } + } + } + Query docValuesQuery = NumericDocValuesField.newSlowSetQuery(field.getName(), points); + if (indexQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, docValuesQuery); + } else { + return docValuesQuery; + } + } else if (indexQuery != null) { + return indexQuery; } else { - return FloatPoint.newSetQuery(field.getName(), values); + return super.getSetQuery(parser, field, externalVals); } } diff --git a/solr/core/src/java/org/apache/solr/schema/IntField.java b/solr/core/src/java/org/apache/solr/schema/IntField.java new file mode 100644 index 00000000000..acfff4bdbd6 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/IntField.java @@ -0,0 +1,349 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValuedIntFieldSource; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; + +/** + * An {@code NumericField} implementation of a field for {@code Int} values using {@code IntPoint}, + * {@code StringField}, {@code SortedNumericDocValuesField} and {@code StoredField}. + * + * @see PointField + * @see IntPoint + */ +public class IntField extends NumericField implements IntValueFieldType { + + public IntField() { + type = NumberType.INTEGER; + } + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).intValue(); + if (val instanceof CharSequence) return Integer.parseInt(val.toString()); + return super.toNativeType(val); + } + + @Override + public Query getPointFieldQuery(QParser parser, SchemaField field, String value) { + return IntPoint.newExactQuery(field.getName(), parseIntFromUser(field.getName(), value)); + } + + @Override + public Query getDocValuesFieldQuery(QParser parser, SchemaField field, String value) { + return SortedNumericDocValuesField.newSlowExactQuery( + field.getName(), parseIntFromUser(field.getName(), value)); + } + + @Override + public Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + int actualMin, actualMax; + if (min == null) { + actualMin = Integer.MIN_VALUE; + } else { + actualMin = parseIntFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Integer.MAX_VALUE) return new MatchNoDocsQuery(); + ++actualMin; + } + } + if (max == null) { + actualMax = Integer.MAX_VALUE; + } else { + actualMax = parseIntFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Integer.MIN_VALUE) return new MatchNoDocsQuery(); + --actualMax; + } + } + return IntPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + int actualMin, actualMax; + if (min == null) { + actualMin = Integer.MIN_VALUE; + } else { + actualMin = parseIntFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Integer.MAX_VALUE) return new MatchNoDocsQuery(); + ++actualMin; + } + } + if (max == null) { + actualMax = Integer.MAX_VALUE; + } else { + actualMax = parseIntFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Integer.MIN_VALUE) return new MatchNoDocsQuery(); + --actualMax; + } + } + return SortedNumericDocValuesField.newSlowRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getPointSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + int[] values = new int[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseIntFromUser(field.getName(), val); + } + return IntPoint.newSetQuery(field.getName(), values); + } + + @Override + public Query getDocValuesSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] points = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + points[i++] = parseIntFromUser(field.getName(), val); + } + return SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return IntPoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final StoredValue storedValue = f.storedValue(); + if (storedValue != null) { + return storedValue.getIntValue(); + } + final Number val = f.numericValue(); + if (val != null) { + return val.intValue(); + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + public String storedToReadable(IndexableField f) { + return Integer.toString(f.storedValue().getIntValue()); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Integer.toString(IntPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Integer.BYTES); + result.setLength(Integer.BYTES); + IntPoint.encodeDimension(parseIntFromUser(null, val.toString()), result.bytes(), 0); + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + return null; + } else { + return Type.INTEGER_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new MultiValuedIntFieldSource(field.getName(), SortedNumericSelector.Type.MIN); + } + + @Override + protected ValueSource getSingleValueSource(SortedNumericSelector.Type choice, SchemaField f) { + return new MultiValuedIntFieldSource(f.getName(), choice); + } + + @Override + public List createFields(SchemaField sf, Object value) { + int intValue = + (value instanceof Number) + ? ((Number) value).intValue() + : Integer.parseInt(value.toString()); + return Collections.singletonList( + new SolrIntField( + sf.getName(), + intValue, + sf.indexed(), + sf.enhancedIndex(), + sf.hasDocValues(), + sf.stored())); + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + int intValue = + (value instanceof Number) + ? ((Number) value).intValue() + : Integer.parseInt(value.toString()); + return new IntPoint(field.getName(), intValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Integer) this.toNativeType(value)); + } + + /** + * Wrapper for {@link Integer#parseInt(String)} that throws a BAD_REQUEST error if the input is + * not valid + * + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static int parseIntFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException( + "Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Integer.parseInt(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + static final class SolrIntField extends Field { + + static org.apache.lucene.document.FieldType getType( + boolean rangeIndex, boolean termIndex, boolean docValues, boolean stored) { + org.apache.lucene.document.FieldType type = new org.apache.lucene.document.FieldType(); + if (rangeIndex) { + type.setDimensions(1, Integer.BYTES); + } + if (termIndex) { + type.setIndexOptions(IndexOptions.DOCS); + } + if (docValues) { + type.setDocValuesType(DocValuesType.SORTED_NUMERIC); + } + type.setTokenized(false); + type.setStored(stored); + type.freeze(); + return type; + } + + private final StoredValue storedValue; + + /** + * Creates a new IntField, indexing the provided point and term, storing it as a DocValue, and + * optionally storing it as a stored field. + * + * @param name field name + * @param value the int value + * @throws IllegalArgumentException if the field name or value is null. + */ + public SolrIntField( + String name, + int value, + boolean rangeIndex, + boolean termIndex, + boolean docValues, + boolean stored) { + super(name, getType(rangeIndex, termIndex, docValues, stored)); + fieldsData = (long) value; + if (stored) { + storedValue = new StoredValue(value); + } else { + storedValue = null; + } + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } + + @Override + public BytesRef binaryValue() { + byte[] encodedPoint = new byte[Integer.BYTES]; + int value = getValueAsInt(); + IntPoint.encodeDimension(value, encodedPoint, 0); + return new BytesRef(encodedPoint); + } + + private int getValueAsInt() { + return numericValue().intValue(); + } + + @Override + public StoredValue storedValue() { + return storedValue; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " <" + name + ':' + getValueAsInt() + '>'; + } + + @Override + public void setIntValue(int value) { + super.setLongValue(value); + if (storedValue != null) { + storedValue.setIntValue(value); + } + } + + @Override + public void setLongValue(long value) { + throw new IllegalArgumentException("cannot change value type from Int to Long"); + } + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/IntPointField.java b/solr/core/src/java/org/apache/solr/schema/IntPointField.java index 5f44195286b..1f600fa41de 100644 --- a/solr/core/src/java/org/apache/solr/schema/IntPointField.java +++ b/solr/core/src/java/org/apache/solr/schema/IntPointField.java @@ -18,13 +18,14 @@ package org.apache.solr.schema; import java.util.Collection; -import org.apache.lucene.document.IntField; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.IntFieldSource; import org.apache.lucene.queries.function.valuesource.MultiValuedIntFieldSource; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortedNumericSelector; @@ -104,26 +105,40 @@ public Object toObject(IndexableField f) { } @Override - protected Query getExactQuery(SchemaField field, String externalVal) { - return IntPoint.newExactQuery(field.getName(), parseIntFromUser(field.getName(), externalVal)); - } - - @Override - public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { - assert externalVal.size() > 0; - if (!field.indexed()) { - return super.getSetQuery(parser, field, externalVal); - } - int[] values = new int[externalVal.size()]; - int i = 0; - for (String val : externalVal) { - values[i] = parseIntFromUser(field.getName(), val); - i++; + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { + assert externalVals.size() > 0; + Query indexQuery = null; + int[] values = null; + if (field.indexed()) { + values = new int[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseIntFromUser(field.getName(), val); + } + indexQuery = IntPoint.newSetQuery(field.getName(), values); } if (field.hasDocValues()) { - return IntField.newSetQuery(field.getName(), values); + long[] points = new long[externalVals.size()]; + if (values != null) { + for (int i = 0; i < values.length; i++) { + points[i] = values[i]; + } + } else { + int i = 0; + for (String val : externalVals) { + points[i++] = parseIntFromUser(field.getName(), val); + } + } + Query docValuesQuery = SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + if (indexQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, docValuesQuery); + } else { + return docValuesQuery; + } + } else if (indexQuery != null) { + return indexQuery; } else { - return IntPoint.newSetQuery(field.getName(), values); + return super.getSetQuery(parser, field, externalVals); } } diff --git a/solr/core/src/java/org/apache/solr/schema/LongField.java b/solr/core/src/java/org/apache/solr/schema/LongField.java new file mode 100644 index 00000000000..90011269129 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/LongField.java @@ -0,0 +1,340 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.schema; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.queries.function.valuesource.MultiValuedLongFieldSource; +import org.apache.lucene.search.MatchNoDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.solr.common.SolrException; +import org.apache.solr.search.QParser; +import org.apache.solr.uninverting.UninvertingReader.Type; + +/** + * An {@code NumericField} implementation of a field for {@code Long} values using {@code + * LongPoint}, {@code StringField}, {@code SortedNumericDocValuesField} and {@code StoredField}. + * + * @see PointField + * @see LongPoint + */ +public class LongField extends NumericField implements LongValueFieldType { + + public LongField() { + type = NumberType.LONG; + } + + @Override + public Object toNativeType(Object val) { + if (val == null) return null; + if (val instanceof Number) return ((Number) val).longValue(); + if (val instanceof CharSequence) return Long.parseLong(val.toString()); + return super.toNativeType(val); + } + + @Override + public Query getPointFieldQuery(QParser parser, SchemaField field, String value) { + return LongPoint.newExactQuery(field.getName(), parseLongFromUser(field.getName(), value)); + } + + @Override + public Query getDocValuesFieldQuery(QParser parser, SchemaField field, String value) { + return SortedNumericDocValuesField.newSlowExactQuery( + field.getName(), parseLongFromUser(field.getName(), value)); + } + + @Override + public Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + long actualMin, actualMax; + if (min == null) { + actualMin = Long.MIN_VALUE; + } else { + actualMin = parseLongFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Long.MAX_VALUE) return new MatchNoDocsQuery(); + ++actualMin; + } + } + if (max == null) { + actualMax = Long.MAX_VALUE; + } else { + actualMax = parseLongFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Long.MIN_VALUE) return new MatchNoDocsQuery(); + --actualMax; + } + } + return LongPoint.newRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + long actualMin, actualMax; + if (min == null) { + actualMin = Long.MIN_VALUE; + } else { + actualMin = parseLongFromUser(field.getName(), min); + if (!minInclusive) { + if (actualMin == Long.MAX_VALUE) return new MatchNoDocsQuery(); + ++actualMin; + } + } + if (max == null) { + actualMax = Long.MAX_VALUE; + } else { + actualMax = parseLongFromUser(field.getName(), max); + if (!maxInclusive) { + if (actualMax == Long.MIN_VALUE) return new MatchNoDocsQuery(); + --actualMax; + } + } + return SortedNumericDocValuesField.newSlowRangeQuery(field.getName(), actualMin, actualMax); + } + + @Override + public Query getPointSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] values = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseLongFromUser(field.getName(), val); + } + return LongPoint.newSetQuery(field.getName(), values); + } + + @Override + public Query getDocValuesSetQuery( + QParser parser, SchemaField field, Collection externalVals) { + long[] points = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + points[i++] = parseLongFromUser(field.getName(), val); + } + return SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + } + + @Override + public Object toObject(SchemaField sf, BytesRef term) { + return LongPoint.decodeDimension(term.bytes, term.offset); + } + + @Override + public Object toObject(IndexableField f) { + final StoredValue storedValue = f.storedValue(); + if (storedValue != null) { + return storedValue.getLongValue(); + } + final Number val = f.numericValue(); + if (val != null) { + return val.longValue(); + } else { + throw new AssertionError("Unexpected state. Field: '" + f + "'"); + } + } + + @Override + public String storedToReadable(IndexableField f) { + return Long.toString(f.storedValue().getLongValue()); + } + + @Override + protected String indexedToReadable(BytesRef indexedForm) { + return Long.toString(LongPoint.decodeDimension(indexedForm.bytes, indexedForm.offset)); + } + + @Override + public void readableToIndexed(CharSequence val, BytesRefBuilder result) { + result.grow(Long.BYTES); + result.setLength(Long.BYTES); + LongPoint.encodeDimension(parseLongFromUser(null, val.toString()), result.bytes(), 0); + } + + @Override + public Type getUninversionType(SchemaField sf) { + if (sf.multiValued()) { + return null; + } else { + return Type.LONG_POINT; + } + } + + @Override + public ValueSource getValueSource(SchemaField field, QParser qparser) { + field.checkFieldCacheSource(); + return new MultiValuedLongFieldSource(field.getName(), SortedNumericSelector.Type.MIN); + } + + @Override + protected ValueSource getSingleValueSource(SortedNumericSelector.Type choice, SchemaField f) { + return new MultiValuedLongFieldSource(f.getName(), choice); + } + + @Override + public List createFields(SchemaField sf, Object value) { + long longValue = + (value instanceof Number) ? ((Number) value).longValue() : Long.parseLong(value.toString()); + return Collections.singletonList( + new SolrLongField( + sf.getName(), + longValue, + sf.indexed(), + sf.enhancedIndex(), + sf.hasDocValues(), + sf.stored())); + } + + @Override + public IndexableField createField(SchemaField field, Object value) { + long longValue = + (value instanceof Number) ? ((Number) value).longValue() : Long.parseLong(value.toString()); + return new LongPoint(field.getName(), longValue); + } + + @Override + protected StoredField getStoredField(SchemaField sf, Object value) { + return new StoredField(sf.getName(), (Long) this.toNativeType(value)); + } + + /** + * Wrapper for {@link Long#parseLong(String)} that throws a BAD_REQUEST error if the input is not + * valid + * + * @param fieldName used in any exception, may be null + * @param val string to parse, NPE if null + */ + static long parseLongFromUser(String fieldName, String val) { + if (val == null) { + throw new NullPointerException( + "Invalid input" + (null == fieldName ? "" : " for field " + fieldName)); + } + try { + return Long.parseLong(val); + } catch (NumberFormatException e) { + String msg = "Invalid Number: " + val + (null == fieldName ? "" : " for field " + fieldName); + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, msg); + } + } + + static final class SolrLongField extends Field { + + static org.apache.lucene.document.FieldType getType( + boolean rangeIndex, boolean termIndex, boolean docValues, boolean stored) { + org.apache.lucene.document.FieldType type = new org.apache.lucene.document.FieldType(); + if (rangeIndex) { + type.setDimensions(1, Long.BYTES); + } + if (termIndex) { + type.setIndexOptions(IndexOptions.DOCS); + } + if (docValues) { + type.setDocValuesType(DocValuesType.SORTED_NUMERIC); + } + type.setTokenized(false); + type.setStored(stored); + type.freeze(); + return type; + } + + private final StoredValue storedValue; + + /** + * Creates a new LongField, indexing the provided point and term, storing it as a DocValue, and + * optionally storing it as a stored field. + * + * @param name field name + * @param value the long value + * @throws IllegalArgumentException if the field name or value is null. + */ + public SolrLongField( + String name, + long value, + boolean rangeIndex, + boolean termIndex, + boolean docValues, + boolean stored) { + super(name, getType(rangeIndex, termIndex, docValues, stored)); + fieldsData = value; + if (stored) { + storedValue = new StoredValue(value); + } else { + storedValue = null; + } + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } + + @Override + public BytesRef binaryValue() { + byte[] encodedPoint = new byte[Long.BYTES]; + long value = getValueAsLong(); + LongPoint.encodeDimension(value, encodedPoint, 0); + return new BytesRef(encodedPoint); + } + + private long getValueAsLong() { + return numericValue().longValue(); + } + + @Override + public StoredValue storedValue() { + return storedValue; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " <" + name + ':' + getValueAsLong() + '>'; + } + + @Override + public void setLongValue(long value) { + super.setLongValue(value); + if (storedValue != null) { + storedValue.setLongValue(value); + } + } + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/LongPointField.java b/solr/core/src/java/org/apache/solr/schema/LongPointField.java index 05a014ae4e3..4afed68c803 100644 --- a/solr/core/src/java/org/apache/solr/schema/LongPointField.java +++ b/solr/core/src/java/org/apache/solr/schema/LongPointField.java @@ -18,13 +18,14 @@ package org.apache.solr.schema; import java.util.Collection; -import org.apache.lucene.document.LongField; import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedNumericDocValuesField; import org.apache.lucene.document.StoredField; import org.apache.lucene.index.IndexableField; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; import org.apache.lucene.queries.function.valuesource.MultiValuedLongFieldSource; +import org.apache.lucene.search.IndexOrDocValuesQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; @@ -103,27 +104,39 @@ public Object toObject(IndexableField f) { } @Override - protected Query getExactQuery(SchemaField field, String externalVal) { - return LongPoint.newExactQuery( - field.getName(), parseLongFromUser(field.getName(), externalVal)); - } - - @Override - public Query getSetQuery(QParser parser, SchemaField field, Collection externalVal) { - assert externalVal.size() > 0; - if (!field.indexed()) { - return super.getSetQuery(parser, field, externalVal); - } - long[] values = new long[externalVal.size()]; - int i = 0; - for (String val : externalVal) { - values[i] = parseLongFromUser(field.getName(), val); - i++; + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { + assert externalVals.size() > 0; + Query indexQuery = null; + long[] values = null; + if (field.indexed()) { + values = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + values[i++] = parseLongFromUser(field.getName(), val); + } + indexQuery = LongPoint.newSetQuery(field.getName(), values); } if (field.hasDocValues()) { - return LongField.newSetQuery(field.getName(), values); + long[] points; + if (values != null) { + points = values.clone(); + } else { + points = new long[externalVals.size()]; + int i = 0; + for (String val : externalVals) { + points[i++] = parseLongFromUser(field.getName(), val); + } + } + Query docValuesQuery = SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points); + if (indexQuery != null) { + return new IndexOrDocValuesQuery(indexQuery, docValuesQuery); + } else { + return docValuesQuery; + } + } else if (indexQuery != null) { + return indexQuery; } else { - return LongPoint.newSetQuery(field.getName(), values); + return super.getSetQuery(parser, field, externalVals); } } diff --git a/solr/core/src/java/org/apache/solr/schema/NumericField.java b/solr/core/src/java/org/apache/solr/schema/NumericField.java new file mode 100644 index 00000000000..c1d06a75b64 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/NumericField.java @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Map; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.queries.function.ValueSource; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.SortedNumericSelector; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.CharsRefBuilder; +import org.apache.lucene.util.NumericUtils; +import org.apache.solr.common.SolrException; +import org.apache.solr.response.TextResponseWriter; +import org.apache.solr.search.QParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides field types to support for Lucene's {@link org.apache.lucene.document.IntPoint}, {@link + * org.apache.lucene.document.LongPoint}, {@link org.apache.lucene.document.FloatPoint} and {@link + * org.apache.lucene.document.DoublePoint}. See {@link org.apache.lucene.search.PointRangeQuery} for + * more details. It supports integer, float, long and double types. See subclasses for details.
+ * {@code DocValues} are supported for single-value cases ({@code NumericDocValues}). {@code + * FieldCache} is not supported for {@code PointField}s, so sorting, faceting, etc on these fields + * require the use of {@code docValues="true"} in the schema. + */ +public abstract class NumericField extends PrimitiveFieldType { + + protected NumberType type; + + /** + * @return the type of this field + */ + @Override + public NumberType getNumberType() { + return type; + } + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + /** + * The Test framework can set this global variable to instruct PointField that (on init) it should + * be tolerant of the precisionStep argument used by TrieFields. This allows for + * simple randomization of TrieFields and PointFields w/o extensive duplication of + * <fieldType/> declarations. + * + *

NOTE: When {@link TrieField} is removed, this boolean must also be removed + * + * @lucene.internal + * @lucene.experimental + */ + public static boolean TEST_HACK_IGNORE_USELESS_TRIEFIELD_ARGS = false; + + /** + * NOTE: This method can be removed completely when {@link + * #TEST_HACK_IGNORE_USELESS_TRIEFIELD_ARGS} is removed + */ + @Override + protected void init(IndexSchema schema, Map args) { + super.init(schema, args); + if (TEST_HACK_IGNORE_USELESS_TRIEFIELD_ARGS) { + args.remove("precisionStep"); + } + } + + @Override + public boolean isPointField() { + return true; + } + + @Override + protected boolean hasIndexedTerms(SchemaField field) { + return field.enhancedIndex(); + } + + @Override + public final ValueSource getSingleValueSource( + MultiValueSelector choice, SchemaField field, QParser parser) { + // trivial base case + if (!field.multiValued()) { + // single value matches any selector + return getValueSource(field, parser); + } + + // Point fields don't support UninvertingReader. See SOLR-9202 + if (!field.hasDocValues()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "docValues='true' is required to select '" + + choice.toString() + + "' value from multivalued field (" + + field.getName() + + ") at query time"); + } + + // multivalued Point fields all use SortedSetDocValues, so we give a clean error if that's + // not supported by the specified choice, else we delegate to a helper + SortedNumericSelector.Type selectorType = choice.getSortedNumericSelectorType(); + if (null == selectorType) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + choice.toString() + + " is not a supported option for picking a single value" + + " from the multivalued field: " + + field.getName() + + " (type: " + + this.getTypeName() + + ")"); + } + + return getSingleValueSource(selectorType, field); + } + + /** + * Helper method that will only be called for multivalued Point fields that have doc values. + * Default impl throws an error indicating that selecting a single value from this multivalued + * field is not supported for this field type + * + * @param choice the selector Type to use, will never be null + * @param field the field to use, guaranteed to be multivalued. + * @see #getSingleValueSource(MultiValueSelector,SchemaField,QParser) + */ + protected abstract ValueSource getSingleValueSource( + SortedNumericSelector.Type choice, SchemaField field); + + @Override + public boolean isTokenized() { + return false; + } + + @Override + public boolean multiValuedFieldCache() { + return false; + } + + @Override + public Query getFieldQuery(QParser parser, SchemaField field, String externalVal) { + if (field.indexed() && field.hasDocValues()) { + Query indexQuery = getIndexFieldQuery(parser, field, externalVal); + Query dvQuery = getDocValuesFieldQuery(parser, field, externalVal); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); + } else if (field.hasDocValues()) { + // currently implemented as singleton range + return getDocValuesFieldQuery(parser, field, externalVal); + } else { + return getIndexFieldQuery(parser, field, externalVal); + } + } + + final Query getIndexFieldQuery(QParser parser, SchemaField field, String externalVal) { + if (hasIndexedTerms(field)) { + BytesRefBuilder br = new BytesRefBuilder(); + readableToIndexed(externalVal, br); + return new TermQuery(new Term(field.getName(), br)); + } else { + return getPointFieldQuery(parser, field, externalVal); + } + } + + public abstract Query getPointFieldQuery(QParser parser, SchemaField field, String externalVal); + + public abstract Query getDocValuesFieldQuery( + QParser parser, SchemaField field, String externalVal); + + @Override + public Query getSetQuery(QParser parser, SchemaField field, Collection externalVals) { + assert !externalVals.isEmpty(); + if (field.indexed() && field.hasDocValues()) { + Query pointsQuery = getIndexSetQuery(parser, field, externalVals); + Query dvQuery = getDocValuesSetQuery(parser, field, externalVals); + return new IndexOrDocValuesQuery(pointsQuery, dvQuery); + } else if (field.hasDocValues()) { + return getDocValuesSetQuery(parser, field, externalVals); + } else if (field.indexed()) { + return getIndexSetQuery(parser, field, externalVals); + } else { + return super.getSetQuery(parser, field, externalVals); + } + } + + final Query getIndexSetQuery(QParser parser, SchemaField field, Collection externalVals) { + if (hasIndexedTerms(field)) { + List lst = new ArrayList<>(externalVals.size()); + BytesRefBuilder br = new BytesRefBuilder(); + for (String externalVal : externalVals) { + readableToIndexed(externalVal, br); + lst.add(br.toBytesRef()); + } + return new TermInSetQuery(field.getName(), lst); + } else if (field.indexed()) { + return getPointSetQuery(parser, field, externalVals); + } + return null; + } + + public abstract Query getPointSetQuery( + QParser parser, SchemaField field, Collection externalVals); + + public abstract Query getDocValuesSetQuery( + QParser parser, SchemaField field, Collection externalVals); + + @Override + protected Query getSpecializedRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive) { + if (field.indexed() && field.hasDocValues()) { + Query pointsQuery = getPointRangeQuery(parser, field, min, max, minInclusive, maxInclusive); + Query dvQuery = getDocValuesRangeQuery(parser, field, min, max, minInclusive, maxInclusive); + return new IndexOrDocValuesQuery(pointsQuery, dvQuery); + } else if (field.hasDocValues()) { + return getDocValuesRangeQuery(parser, field, min, max, minInclusive, maxInclusive); + } else { + return getPointRangeQuery(parser, field, min, max, minInclusive, maxInclusive); + } + } + + public abstract Query getPointRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive); + + public abstract Query getDocValuesRangeQuery( + QParser parser, + SchemaField field, + String min, + String max, + boolean minInclusive, + boolean maxInclusive); + + @Override + public String storedToReadable(IndexableField f) { + return toExternal(f); + } + + @Override + public String toInternal(String val) { + return toInternalByteRef(val).utf8ToString(); + } + + public BytesRef toInternalByteRef(String val) { + final BytesRefBuilder bytes = new BytesRefBuilder(); + readableToIndexed(val, bytes); + return bytes.get(); + } + + @Override + public void write(TextResponseWriter writer, String name, IndexableField f) throws IOException { + writer.writeVal(name, toObject(f)); + } + + @Override + public String storedToIndexed(IndexableField f) { + throw new UnsupportedOperationException("Not supported with PointFields"); + } + + @Override + public CharsRef indexedToReadable(BytesRef indexedForm, CharsRefBuilder charsRef) { + final String value = indexedToReadable(indexedForm); + charsRef.grow(value.length()); + charsRef.setLength(value.length()); + value.getChars(0, charsRef.length(), charsRef.chars(), 0); + return charsRef.get(); + } + + @Override + public String indexedToReadable(String indexedForm) { + return indexedToReadable(new BytesRef(indexedForm)); + } + + protected abstract String indexedToReadable(BytesRef indexedForm); + + @Override + public Query getPrefixQuery(QParser parser, SchemaField sf, String termStr) { + if (termStr != null && termStr.isEmpty()) { + return getExistenceQuery(parser, sf); + } + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, "Can't run prefix queries on numeric fields"); + } + + protected boolean isFieldUsed(SchemaField field) { + boolean indexed = field.indexed(); + boolean stored = field.stored(); + boolean docValues = field.hasDocValues(); + + if (!indexed && !stored && !docValues) { + log.trace("Ignoring unindexed/unstored field: {}", field); + return false; + } + return true; + } + + @Override + public List createFields(SchemaField sf, Object value) { + if (!isFieldUsed(sf)) { + return Collections.emptyList(); + } + List fields = new ArrayList<>(3); + IndexableField field = null; + if (sf.indexed()) { + field = createField(sf, value); + fields.add(field); + if (sf.enhancedIndex()) { + fields.add(new StringField(sf.getName(), field.binaryValue(), Field.Store.NO)); + } + } + + if (sf.hasDocValues()) { + final Number numericValue; + if (field == null) { + final Object nativeTypeObject = toNativeType(value); + if (getNumberType() == NumberType.DATE) { + numericValue = ((Date) nativeTypeObject).getTime(); + } else { + numericValue = (Number) nativeTypeObject; + } + } else { + numericValue = field.numericValue(); + } + final long bits; + // MultiValued + if (numericValue instanceof Integer || numericValue instanceof Long) { + bits = numericValue.longValue(); + } else if (numericValue instanceof Float) { + bits = NumericUtils.floatToSortableInt(numericValue.floatValue()); + } else { + assert numericValue instanceof Double; + bits = NumericUtils.doubleToSortableLong(numericValue.doubleValue()); + } + fields.add(new SortedNumericDocValuesField(sf.getName(), bits)); + } + if (sf.stored()) { + fields.add(getStoredField(sf, value)); + } + return fields; + } + + protected abstract StoredField getStoredField(SchemaField sf, Object value); + + @Override + public SortField getSortField(SchemaField field, boolean top) { + return getNumericSort(field, getNumberType(), top); + } +} diff --git a/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java b/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java index b3890114e6e..967fcfb4296 100644 --- a/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/NumericFieldType.java @@ -56,7 +56,7 @@ protected Query getDocValuesRangeQuery( String max, boolean minInclusive, boolean maxInclusive) { - assert field.hasDocValues() && (field.getType().isPointField() || !field.multiValued()); + assert field.hasDocValues(); switch (getNumberType()) { case INTEGER: diff --git a/solr/core/src/java/org/apache/solr/schema/PointField.java b/solr/core/src/java/org/apache/solr/schema/PointField.java index 611dcfc43a4..cde97cfb5f9 100644 --- a/solr/core/src/java/org/apache/solr/schema/PointField.java +++ b/solr/core/src/java/org/apache/solr/schema/PointField.java @@ -87,6 +87,11 @@ public boolean isPointField() { return true; } + @Override + protected boolean hasIndexedTerms(SchemaField field) { + return false; + } + @Override public final ValueSource getSingleValueSource( MultiValueSelector choice, SchemaField field, QParser parser) { @@ -158,16 +163,14 @@ public Query getFieldQuery(QParser parser, SchemaField field, String externalVal // currently implemented as singleton range return getRangeQuery(parser, field, externalVal, externalVal, true, true); } else if (field.indexed() && field.hasDocValues()) { - Query pointsQuery = getExactQuery(field, externalVal); + Query indexQuery = getPointRangeQuery(parser, field, externalVal, externalVal, true, true); Query dvQuery = getDocValuesRangeQuery(parser, field, externalVal, externalVal, true, true); - return new IndexOrDocValuesQuery(pointsQuery, dvQuery); + return new IndexOrDocValuesQuery(indexQuery, dvQuery); } else { - return getExactQuery(field, externalVal); + return getPointRangeQuery(parser, field, externalVal, externalVal, true, true); } } - protected abstract Query getExactQuery(SchemaField field, String externalVal); - @Override protected Query getSpecializedRangeQuery( QParser parser, diff --git a/solr/core/src/java/org/apache/solr/schema/SchemaField.java b/solr/core/src/java/org/apache/solr/schema/SchemaField.java index 2a413bf3729..3c9c882d060 100644 --- a/solr/core/src/java/org/apache/solr/schema/SchemaField.java +++ b/solr/core/src/java/org/apache/solr/schema/SchemaField.java @@ -103,6 +103,10 @@ public boolean indexed() { return (properties & INDEXED) != 0; } + public boolean enhancedIndex() { + return (properties & ENHANCED_INDEX) != 0 && indexed(); + } + @Override public boolean stored() { return (properties & STORED) != 0; diff --git a/solr/core/src/java/org/apache/solr/schema/SolrFloatField.java b/solr/core/src/java/org/apache/solr/schema/SolrFloatField.java new file mode 100644 index 00000000000..21d2626ad9b --- /dev/null +++ b/solr/core/src/java/org/apache/solr/schema/SolrFloatField.java @@ -0,0 +1,97 @@ +package org.apache.solr.schema; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.InvertableType; +import org.apache.lucene.document.StoredValue; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NumericUtils; + +public final class SolrFloatField extends Field { + + static org.apache.lucene.document.FieldType getType( + boolean rangeIndex, boolean termIndex, boolean docValues, boolean stored) { + org.apache.lucene.document.FieldType type = new org.apache.lucene.document.FieldType(); + if (rangeIndex) { + type.setDimensions(1, Float.BYTES); + } + if (termIndex) { + type.setIndexOptions(IndexOptions.DOCS); + } + if (docValues) { + type.setDocValuesType(DocValuesType.SORTED_NUMERIC); + } + type.setTokenized(false); + type.setStored(stored); + type.freeze(); + return type; + } + + private final StoredValue storedValue; + + /** + * Creates a new FloatField, indexing the provided point, storing it as a DocValue, and optionally + * storing it as a stored field. + * + * @param name field name + * @param value the float value + * @throws IllegalArgumentException if the field name or value is null. + */ + public SolrFloatField( + String name, + float value, + boolean rangeIndex, + boolean termIndex, + boolean docValues, + boolean stored) { + super(name, getType(rangeIndex, termIndex, docValues, stored)); + fieldsData = (long) NumericUtils.floatToSortableInt(value); + if (stored) { + storedValue = new StoredValue(value); + } else { + storedValue = null; + } + } + + @Override + public InvertableType invertableType() { + return InvertableType.BINARY; + } + + @Override + public BytesRef binaryValue() { + byte[] encodedPoint = new byte[Float.BYTES]; + float value = getValueAsFloat(); + FloatPoint.encodeDimension(value, encodedPoint, 0); + return new BytesRef(encodedPoint); + } + + private float getValueAsFloat() { + return NumericUtils.sortableIntToFloat(numericValue().intValue()); + } + + @Override + public StoredValue storedValue() { + return storedValue; + } + + @Override + public String toString() { + return getClass().getSimpleName() + " <" + name + ':' + getValueAsFloat() + '>'; + } + + @Override + public void setFloatValue(float value) { + super.setLongValue(NumericUtils.floatToSortableInt(value)); + if (storedValue != null) { + storedValue.setFloatValue(value); + } + } + + @Override + public void setLongValue(long value) { + throw new IllegalArgumentException("cannot change value type from Float to Long"); + } +} diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index 9f127f7d460..128458e06e8 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -864,7 +864,8 @@ public ScoreMode scoreMode() { protected void doSetNextReader(LeafReaderContext context) throws IOException { this.contexts[context.ord] = context; this.docBase = context.docBase; - this.collapseValues = DocValues.getNumeric(context.reader(), this.field); + this.collapseValues = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), this.field)); } @Override @@ -946,7 +947,9 @@ public void complete() throws IOException { int currentContext = 0; int currentDocBase = 0; - collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field); + collapseValues = + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(contexts[currentContext].reader(), this.field)); int nextDocBase = currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; leafDelegate = delegate.getLeafCollector(contexts[currentContext]); @@ -965,7 +968,9 @@ public void complete() throws IOException { currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; leafDelegate = delegate.getLeafCollector(contexts[currentContext]); leafDelegate.setScorer(dummy); - collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field); + collapseValues = + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(contexts[currentContext].reader(), this.field)); } final int contextDoc = globalDoc - currentDocBase; @@ -1414,7 +1419,9 @@ public void doSetNextReader(LeafReaderContext context) throws IOException { this.contexts[context.ord] = context; this.docBase = context.docBase; this.collapseStrategy.setNextReader(context); - this.collapseValues = DocValues.getNumeric(context.reader(), this.collapseField); + this.collapseValues = + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(context.reader(), this.collapseField)); } @Override @@ -1449,7 +1456,8 @@ public void complete() throws IOException { int currentContext = 0; int currentDocBase = 0; this.collapseValues = - DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(contexts[currentContext].reader(), this.collapseField)); int nextDocBase = currentContext + 1 < contexts.length ? contexts[currentContext + 1].docBase : maxDoc; leafDelegate = delegate.getLeafCollector(contexts[currentContext]); @@ -1475,7 +1483,9 @@ public void complete() throws IOException { leafDelegate = delegate.getLeafCollector(contexts[currentContext]); leafDelegate.setScorer(dummy); this.collapseValues = - DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField); + DocValues.unwrapSingleton( + DocValues.getSortedNumeric( + contexts[currentContext].reader(), this.collapseField)); } final int contextDoc = globalDoc - currentDocBase; @@ -1791,7 +1801,8 @@ public BlockIntScoreCollector( @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); - this.segmentValues = DocValues.getNumeric(context.reader(), collapseField); + this.segmentValues = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), collapseField)); } @Override @@ -2025,7 +2036,8 @@ public BlockIntSortSpecCollector( @Override protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); - this.segmentValues = DocValues.getNumeric(context.reader(), collapseField); + this.segmentValues = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), collapseField)); } @Override @@ -2401,7 +2413,8 @@ public OrdIntStrategy( @Override public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); + this.minMaxValues = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), this.field)); } @Override @@ -2478,7 +2491,8 @@ public OrdFloatStrategy( @Override public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxValues = DocValues.getNumeric(context.reader(), this.field); + this.minMaxValues = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), this.field)); } @Override @@ -2557,7 +2571,8 @@ public OrdLongStrategy( @Override public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); + this.minMaxVals = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), this.field)); } @Override @@ -2946,7 +2961,8 @@ public IntIntStrategy( @Override public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); + this.minMaxVals = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), this.field)); } private int advanceAndGetCurrentVal(int contextDoc) throws IOException { @@ -3040,7 +3056,8 @@ public IntFloatStrategy( @Override public void setNextReader(LeafReaderContext context) throws IOException { - this.minMaxVals = DocValues.getNumeric(context.reader(), this.field); + this.minMaxVals = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(context.reader(), this.field)); } private float advanceAndGetCurrentVal(int contextDoc) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/search/DocValuesIteratorCache.java b/solr/core/src/java/org/apache/solr/search/DocValuesIteratorCache.java index eba8a731881..690e526531f 100644 --- a/solr/core/src/java/org/apache/solr/search/DocValuesIteratorCache.java +++ b/solr/core/src/java/org/apache/solr/search/DocValuesIteratorCache.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.function.Function; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.DocValuesType; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReader; @@ -45,19 +46,19 @@ public class DocValuesIteratorCache { funcMap = new EnumMap<>(DocValuesType.class); static { - funcMap.put(DocValuesType.NUMERIC, LeafReader::getNumericDocValues); - funcMap.put(DocValuesType.BINARY, LeafReader::getBinaryDocValues); + funcMap.put(DocValuesType.NUMERIC, DocValues::getNumeric); + funcMap.put(DocValuesType.BINARY, DocValues::getBinary); funcMap.put( DocValuesType.SORTED, (r, f) -> { - SortedDocValues dvs = r.getSortedDocValues(f); + SortedDocValues dvs = DocValues.getSorted(r, f); return dvs == null || dvs.getValueCount() < 1 ? null : dvs; }); - funcMap.put(DocValuesType.SORTED_NUMERIC, LeafReader::getSortedNumericDocValues); + funcMap.put(DocValuesType.SORTED_NUMERIC, DocValues::getSortedNumeric); funcMap.put( DocValuesType.SORTED_SET, (r, f) -> { - SortedSetDocValues dvs = r.getSortedSetDocValues(f); + SortedSetDocValues dvs = DocValues.getSortedSet(r, f); return dvs == null || dvs.getValueCount() < 1 ? null : dvs; }); } @@ -168,7 +169,7 @@ private DocIdSetIterator getDocValues( if (min == -1) { // we are not yet initialized for this field/leaf. dv = dvFunction.apply(leafReader, schemaField.getName()); - if (dv == null) { + if (dv == null || dv.cost() == 0) { minLocalIds[leafOrd] = DocIdSetIterator.NO_MORE_DOCS; // cache absence of this field return null; } diff --git a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java index 6ba9a7d66a4..3644cb37070 100644 --- a/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/HashQParserPlugin.java @@ -95,7 +95,8 @@ public LongValues getValues(LeafReaderContext ctx, DoubleValues scores) throws I final LongValues[] resultValues = new LongValues[fields.length]; for (int i = 0; i < fields.length; i++) { final String field = fields[i]; - final NumericDocValues numericDocValues = ctx.reader().getNumericDocValues(field); + final NumericDocValues numericDocValues = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(ctx.reader(), field)); if (numericDocValues != null) { // Numeric resultValues[i] = diff --git a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java index eba8327360a..7aa116059e4 100644 --- a/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/IGainTermsQParserPlugin.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Objects; import java.util.TreeSet; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -123,7 +124,7 @@ public IGainTermsCollector( protected void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); LeafReader reader = context.reader(); - leafOutcomeValue = reader.getNumericDocValues(outcome); + leafOutcomeValue = DocValues.unwrapSingleton(DocValues.getSortedNumeric(reader, outcome)); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java index d1a3f3ddef9..51894c7d07a 100644 --- a/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java +++ b/solr/core/src/java/org/apache/solr/search/SolrDocumentFetcher.java @@ -661,10 +661,7 @@ private Object decodeDVField( // return immediately if the number is not decodable, hence won't return an empty list. if (value == null) { return null; - } - // normally never true but LatLonPointSpatialField uses SORTED_NUMERIC even when single - // valued - else if (e.schemaField.multiValued() == false) { + } else if (e.schemaField.multiValued() == false) { return value; } else { outValues.add(value); diff --git a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java index 939f2f64ede..90bfbb2f18d 100644 --- a/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TermsQParserPlugin.java @@ -53,6 +53,7 @@ import org.apache.solr.common.params.SolrParams; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.NumericField; import org.apache.solr.schema.PointField; import org.apache.solr.util.SolrDefaultScorerSupplier; import org.slf4j.Logger; @@ -170,8 +171,13 @@ public Query parse() throws SyntaxError { "Method '%s' not supported in TermsQParser when using PointFields", localParams.get(METHOD))); } - return ((PointField) ft) - .getSetQuery(this, req.getSchema().getField(fname), Arrays.asList(splitVals)); + if (ft instanceof PointField pointField) { + return pointField.getSetQuery( + this, req.getSchema().getField(fname), Arrays.asList(splitVals)); + } else if (ft instanceof NumericField numericField) { + return numericField.getSetQuery( + this, req.getSchema().getField(fname), Arrays.asList(splitVals)); + } } BytesRef[] bytesRefs = new BytesRef[splitVals.length]; diff --git a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java index 0f8142da399..343edc49e2d 100644 --- a/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/TextLogisticRegressionQParserPlugin.java @@ -23,6 +23,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.index.DocValues; import org.apache.lucene.index.LeafReader; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.NumericDocValues; @@ -147,7 +148,8 @@ private static class TextLogisticRegressionCollector extends DelegatingCollector public void doSetNextReader(LeafReaderContext context) throws IOException { super.doSetNextReader(context); leafReader = context.reader(); - leafOutcomeValue = leafReader.getNumericDocValues(trainingParams.outcome); + leafOutcomeValue = + DocValues.unwrapSingleton(DocValues.getSortedNumeric(leafReader, trainingParams.outcome)); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java b/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java index db1fa8f2646..001f5271aba 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java +++ b/solr/core/src/java/org/apache/solr/search/facet/DocValuesAcc.java @@ -70,7 +70,9 @@ public NumericDVAcc(FacetContext fcontext, SchemaField sf) throws IOException { @Override public void setNextReader(LeafReaderContext readerContext) throws IOException { super.setNextReader(readerContext); - values = DocValues.getNumeric(readerContext.reader(), sf.getName()); + values = + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(readerContext.reader(), sf.getName())); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java index 50780dad141..aa551a4ca45 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByArrayDV.java @@ -53,17 +53,13 @@ class FacetFieldProcessorByArrayDV extends FacetFieldProcessorByArray { @Override protected void findStartAndEndOrds() throws IOException { - if (multiValuedField) { - si = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf, null); - if (si instanceof MultiDocValues.MultiSortedSetDocValues) { - ordinalMap = ((MultiDocValues.MultiSortedSetDocValues) si).mapping; - } + si = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf); + if (si instanceof MultiDocValues.MultiSortedSetDocValues multi) { + ordinalMap = multi.mapping; } else { - // multi-valued view - SortedDocValues single = FieldUtil.getSortedDocValues(fcontext.qcontext, sf, null); - si = DocValues.singleton(single); - if (single instanceof MultiDocValues.MultiSortedDocValues) { - ordinalMap = ((MultiDocValues.MultiSortedDocValues) single).mapping; + SortedDocValues single = DocValues.unwrapSingleton(si); + if (single instanceof MultiDocValues.MultiSortedDocValues multi) { + ordinalMap = multi.mapping; } } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java index a61230e9242..68a50897141 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetFieldProcessorByHashDV.java @@ -24,7 +24,6 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.index.MultiDocValues; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.search.ScoreMode; @@ -344,7 +343,7 @@ private void collectDocs() throws IOException { if (calc instanceof TermOrdCalc) { // Strings // TODO support SortedSetDocValues - SortedDocValues globalDocValues = FieldUtil.getSortedDocValues(fcontext.qcontext, sf, null); + SortedDocValues globalDocValues = FieldUtil.getSortedDocValues(fcontext.qcontext, sf); ((TermOrdCalc) calc).lookupOrdFunction = ord -> { try { @@ -385,68 +384,40 @@ public void collect(int segDoc) throws IOException { }); } else { // Numeric: + DocSetUtil.collectSortedDocSet( + fcontext.base, + fcontext.searcher.getIndexReader(), + new SimpleCollector() { + SortedNumericDocValues values = null; // NN - if (sf.multiValued()) { - DocSetUtil.collectSortedDocSet( - fcontext.base, - fcontext.searcher.getIndexReader(), - new SimpleCollector() { - SortedNumericDocValues values = null; // NN - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } - @Override - protected void doSetNextReader(LeafReaderContext ctx) throws IOException { - setNextReaderFirstPhase(ctx); - values = DocValues.getSortedNumeric(ctx.reader(), sf.getName()); - } + @Override + protected void doSetNextReader(LeafReaderContext ctx) throws IOException { + setNextReaderFirstPhase(ctx); + values = DocValues.getSortedNumeric(ctx.reader(), sf.getName()); + } - @Override - public void collect(int segDoc) throws IOException { - if (values.advanceExact(segDoc)) { - long l = values.nextValue(); // This document must have at least one value - collectValFirstPhase(segDoc, l); - for (int i = 1, count = values.docValueCount(); i < count; i++) { - long lnew = values.nextValue(); - // Skip the value if it's equal to the last one, we don't want to double-count - // it - if (lnew != l) { - collectValFirstPhase(segDoc, lnew); - } - l = lnew; + @Override + public void collect(int segDoc) throws IOException { + if (values.advanceExact(segDoc)) { + long l = values.nextValue(); // This document must have at least one value + collectValFirstPhase(segDoc, l); + for (int i = 1, count = values.docValueCount(); i < count; i++) { + long lnew = values.nextValue(); + // Skip the value if it's equal to the last one, we don't want to double-count + // it + if (lnew != l) { + collectValFirstPhase(segDoc, lnew); } + l = lnew; } } - }); - } else { - DocSetUtil.collectSortedDocSet( - fcontext.base, - fcontext.searcher.getIndexReader(), - new SimpleCollector() { - NumericDocValues values = null; // NN - - @Override - public ScoreMode scoreMode() { - return ScoreMode.COMPLETE_NO_SCORES; - } - - @Override - protected void doSetNextReader(LeafReaderContext ctx) throws IOException { - setNextReaderFirstPhase(ctx); - values = DocValues.getNumeric(ctx.reader(), sf.getName()); - } - - @Override - public void collect(int segDoc) throws IOException { - if (values.advanceExact(segDoc)) { - collectValFirstPhase(segDoc, values.longValue()); - } - } - }); - } + } + }); } } diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java index a3d7165ea46..2ddede0818b 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRangeProcessor.java @@ -38,6 +38,7 @@ import org.apache.solr.schema.EnumFieldType.EnumMapping; import org.apache.solr.schema.ExchangeRateProvider; import org.apache.solr.schema.FieldType; +import org.apache.solr.schema.NumericField; import org.apache.solr.schema.SchemaField; import org.apache.solr.schema.TrieDateField; import org.apache.solr.schema.TrieField; @@ -740,7 +741,8 @@ private static class FloatCalc extends Calc { @SuppressWarnings("rawtypes") @Override public Comparable bitsToValue(long bits) { - if (field.getType().isPointField() && field.multiValued()) { + if ((field.getType().isPointField() && field.multiValued()) + || field.getType() instanceof NumericField) { return NumericUtils.sortableIntToFloat((int) bits); } else { return Float.intBitsToFloat((int) bits); @@ -771,7 +773,8 @@ private static class DoubleCalc extends Calc { @Override @SuppressWarnings({"rawtypes"}) public Comparable bitsToValue(long bits) { - if (field.getType().isPointField() && field.multiValued()) { + if ((field.getType().isPointField() && field.multiValued()) + || field.getType() instanceof NumericField) { return NumericUtils.sortableLongToDouble(bits); } else { return Double.longBitsToDouble(bits); diff --git a/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java b/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java index 7948dc40206..c26ae727bb9 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java +++ b/solr/core/src/java/org/apache/solr/search/facet/FieldUtil.java @@ -18,19 +18,12 @@ import java.io.IOException; import org.apache.lucene.index.DocValues; -import org.apache.lucene.index.DocValuesType; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.LeafReader; -import org.apache.lucene.index.NumericDocValues; import org.apache.lucene.index.SortedDocValues; -import org.apache.lucene.index.SortedNumericDocValues; import org.apache.lucene.index.SortedSetDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.solr.schema.SchemaField; -import org.apache.solr.search.QParser; import org.apache.solr.search.QueryContext; -import org.apache.solr.search.SolrIndexSearcher; import org.apache.solr.uninverting.FieldCacheImpl; /** @@ -38,59 +31,15 @@ */ public class FieldUtil { - /** Simpler method that creates a request context and looks up the field for you */ - public static SortedDocValues getSortedDocValues(SolrIndexSearcher searcher, String field) + public static SortedDocValues getSortedDocValues(QueryContext context, SchemaField field) throws IOException { - SchemaField sf = searcher.getSchema().getField(field); - QueryContext qContext = QueryContext.newContext(searcher); - return getSortedDocValues(qContext, sf, null); + return DocValues.unwrapSingleton( + DocValues.getSortedSet(context.searcher().getSlowAtomicReader(), field.getName())); } - public static SortedDocValues getSortedDocValues( - QueryContext context, SchemaField field, QParser qparser) throws IOException { - var reader = context.searcher().getSlowAtomicReader(); - var dv = reader.getSortedDocValues(field.getName()); - checkDvType(dv, field, reader); - return dv == null ? DocValues.emptySorted() : dv; - } - - public static SortedSetDocValues getSortedSetDocValues( - QueryContext context, SchemaField field, QParser qparser) throws IOException { - var reader = context.searcher().getSlowAtomicReader(); - var dv = reader.getSortedSetDocValues(field.getName()); - checkDvType(dv, field, reader); - return dv == null ? DocValues.emptySortedSet() : dv; - } - - public static NumericDocValues getNumericDocValues( - QueryContext context, SchemaField field, QParser qparser) throws IOException { - var reader = context.searcher().getSlowAtomicReader(); - var dv = reader.getNumericDocValues(field.getName()); - checkDvType(dv, field, reader); - return dv == null ? DocValues.emptyNumeric() : dv; - } - - private static void checkDvType(Object dv, SchemaField field, LeafReader reader) { - if (dv == null) { - return; - } - FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field.getName()); - if (fieldInfo == null) { - return; - } - var dvType = fieldInfo.getDocValuesType(); - if (dvType == DocValuesType.NONE) { - return; - } else if (dvType == DocValuesType.SORTED) { - if (dv instanceof SortedDocValues) return; - } else if (dvType == DocValuesType.SORTED_SET) { - if (dv instanceof SortedSetDocValues) return; - } else if (dvType == DocValuesType.NUMERIC) { - if (dv instanceof NumericDocValues) return; - } else if (dvType == DocValuesType.SORTED_NUMERIC) { - if (dv instanceof SortedNumericDocValues) return; - } - throw new IllegalStateException("Unexpected DocValues type " + dvType + " for field " + field); + public static SortedSetDocValues getSortedSetDocValues(QueryContext context, SchemaField field) + throws IOException { + return DocValues.getSortedSet(context.searcher().getSlowAtomicReader(), field.getName()); } /** diff --git a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java index ca4ecfa6d02..77d8ad84ca9 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java +++ b/solr/core/src/java/org/apache/solr/search/facet/HLLAgg.java @@ -193,7 +193,9 @@ public NumericAcc(FacetContext fcontext, String field, int numSlots) throws IOEx @Override public void setNextReader(LeafReaderContext readerContext) throws IOException { super.setNextReader(readerContext); - values = DocValues.getNumeric(readerContext.reader(), sf.getName()); + values = + DocValues.unwrapSingleton( + DocValues.getSortedNumeric(readerContext.reader(), sf.getName())); } @Override diff --git a/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java b/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java index 2fbe07bd46d..93ba8b2ff8e 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java +++ b/solr/core/src/java/org/apache/solr/search/facet/MinMaxAgg.java @@ -425,7 +425,7 @@ public SingleValuedOrdAcc(FacetContext fcontext, SchemaField field, int numSlots @Override public void resetIterators() throws IOException { super.resetIterators(); - topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null); + topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field); if (topLevel instanceof MultiDocValues.MultiSortedDocValues) { ordMap = ((MultiDocValues.MultiSortedDocValues) topLevel).mapping; subDvs = ((MultiDocValues.MultiSortedDocValues) topLevel).values; @@ -485,7 +485,7 @@ public MinMaxSortedSetDVAcc(FacetContext fcontext, SchemaField field, int numSlo @Override public void resetIterators() throws IOException { super.resetIterators(); - topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf, null); + topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf); if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) { ordMap = ((MultiDocValues.MultiSortedSetDocValues) topLevel).mapping; subDvs = ((MultiDocValues.MultiSortedSetDocValues) topLevel).values; diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java index d5cd7bf7f8e..b8702b99b3d 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java +++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueMultiDvSlotAcc.java @@ -43,7 +43,7 @@ public UniqueMultiDvSlotAcc( @Override public void resetIterators() throws IOException { - topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, field, null); + topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, field); nTerms = (int) topLevel.getValueCount(); if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) { ordMap = ((MultiDocValues.MultiSortedSetDocValues) topLevel).mapping; diff --git a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java index dd51050e4ca..0bd1aacfc1b 100644 --- a/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java +++ b/solr/core/src/java/org/apache/solr/search/facet/UniqueSinglevaluedSlotAcc.java @@ -44,7 +44,7 @@ public UniqueSinglevaluedSlotAcc( @Override public void resetIterators() throws IOException { super.resetIterators(); - topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null); + topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field); nTerms = topLevel.getValueCount(); if (topLevel instanceof MultiDocValues.MultiSortedDocValues) { ordMap = ((MultiDocValues.MultiSortedDocValues) topLevel).mapping; diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-numeric-full.xml b/solr/core/src/test-files/solr/collection1/conf/schema-numeric-full.xml new file mode 100644 index 00000000000..306326bb4c3 --- /dev/null +++ b/solr/core/src/test-files/solr/collection1/conf/schema-numeric-full.xml @@ -0,0 +1,247 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + id + + diff --git a/solr/core/src/test/org/apache/solr/schema/TestNumericFields.java b/solr/core/src/test/org/apache/solr/schema/TestNumericFields.java new file mode 100644 index 00000000000..214087f14d8 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/schema/TestNumericFields.java @@ -0,0 +1,6410 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.schema; + +import java.io.IOException; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.SortedSet; +import java.util.TreeSet; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.DoublePoint; +import org.apache.lucene.document.FloatPoint; +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LeafReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.PointValues; +import org.apache.lucene.index.StoredFields; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexOrDocValuesQuery; +import org.apache.lucene.search.PointInSetQuery; +import org.apache.lucene.search.PointRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermInSetQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.index.SlowCompositeReaderWrapper; +import org.apache.solr.schema.IndexSchema.DynamicField; +import org.apache.solr.search.SolrQueryParser; +import org.apache.solr.util.DateMathParser; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; + +/** Tests for NumericField functionality */ +public class TestNumericFields extends SolrTestCaseJ4 { + + // long overflow can occur in some date calculations if gaps are too large, so we limit to a + // million years BC & AD. + private static final long MIN_DATE_EPOCH_MILLIS = + LocalDateTime.parse("-1000000-01-01T00:00:00") + .toInstant(ZoneOffset.ofHours(0)) + .toEpochMilli(); + private static final long MAX_DATE_EPOCH_MILLIS = + LocalDateTime.parse("+1000000-01-01T00:00:00") + .toInstant(ZoneOffset.ofHours(0)) + .toEpochMilli(); + + private static final String[] FIELD_SUFFIXES = + new String[] { + "", + "_dv", + "_mv", + "_mv_dv", + "_e", + "_e_dv", + "_e_mv", + "_e_mv_dv", + "_ni", + "_ni_dv", + "_ni_dv_ns", + "_ni_dv_ns_mv", + "_ni_mv", + "_ni_mv_dv", + "_ni_ns", + "_ni_ns_mv", + "_dv_ns", + "_ni_ns_dv", + "_dv_ns_mv", + "_smf", + "_e_smf", + "_e_dv_smf", + "_e_mv_smf", + "_e_mv_dv_smf", + "_dv_smf", + "_mv_smf", + "_mv_dv_smf", + "_ni_dv_smf", + "_ni_mv_dv_smf", + "_sml", + "_e_sml", + "_e_dv_sml", + "_e_mv_sml", + "_e_mv_dv_sml", + "_dv_sml", + "_mv_sml", + "_mv_dv_sml", + "_ni_dv_sml", + "_ni_mv_dv_sml" + }; + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema-numeric-full.xml"); + } + + @Override + @After + public void tearDown() throws Exception { + clearIndex(); + assertU(commit()); + super.tearDown(); + } + + @Test + public void testIntFieldExactQuery() throws Exception { + doTestIntFieldExactQuery("number_p_i", false); + doTestIntFieldExactQuery("number_p_i_mv", false); + doTestIntFieldExactQuery("number_p_i_dv", false); + doTestIntFieldExactQuery("number_p_i_mv_dv", false); + doTestIntFieldExactQuery("number_p_i_ni_dv", false); + doTestIntFieldExactQuery("number_p_i_ni_ns_dv", false); + doTestIntFieldExactQuery("number_p_i_ni_mv_dv", false); + doTestIntFieldExactQuery("number_p_i_e", false); + doTestIntFieldExactQuery("number_p_i_e_mv", false); + doTestIntFieldExactQuery("number_p_i_e_dv", false); + doTestIntFieldExactQuery("number_p_i_e_mv_dv", false); + } + + @Test + public void testIntFieldNonSearchableExactQuery() throws Exception { + doTestIntFieldExactQuery("number_p_i_ni", false, false); + doTestIntFieldExactQuery("number_p_i_ni_ns", false, false); + } + + @Test + public void testIntFieldReturn() throws Exception { + int numValues = 10 * RANDOM_MULTIPLIER; + String[] ints = toStringArray(getRandomInts(numValues, false)); + doTestFieldReturn("number_p_i", "int", ints); + doTestFieldReturn("number_p_i_dv_ns", "int", ints); + doTestFieldReturn("number_p_i_ni", "int", ints); + } + + @Test + public void testIntFieldRangeQuery() throws Exception { + doTestIntFieldRangeQuery("number_p_i", "int", false); + doTestIntFieldRangeQuery("number_p_i_ni_ns_dv", "int", false); + doTestIntFieldRangeQuery("number_p_i_dv", "int", false); + doTestIntFieldRangeQuery("number_p_i_e", "int", false); + doTestIntFieldRangeQuery("number_p_i_e_dv", "int", false); + } + + @Test + public void testIntFieldNonSearchableRangeQuery() throws Exception { + doTestFieldNonSearchableRangeQuery("number_p_i_ni", toStringArray(getRandomInts(1, false))); + doTestFieldNonSearchableRangeQuery("number_p_i_ni_ns", toStringArray(getRandomInts(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldNonSearchableRangeQuery( + "number_p_i_ni_ns_mv", toStringArray(getRandomInts(numValues, false))); + } + + @Test + public void testIntFieldSortAndFunction() throws Exception { + + final SortedSet regexToTest = dynFieldRegexesForType(IntField.class); + final List sequential = Arrays.asList("0", "1", "2", "3", "4", "5", "6", "7", "8", "9"); + final List randomInts = getRandomInts(10, false); + final List randomIntsMissing = getRandomInts(10, true); + + for (String r : + Arrays.asList( + "*_p_i", + "*_p_i_e", + "*_p_i_e_dv", + "*_p_i_dv", + "*_p_i_dv_ns", + "*_p_i_ni_dv", + "*_p_i_ni_dv_ns", + "*_p_i_ni_ns_dv")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomInts); + doTestIntFieldFunctionQuery(field); + } + for (String r : + Arrays.asList( + "*_p_i_smf", + "*_p_i_e_smf", + "*_p_i_e_dv_smf", + "*_p_i_dv_smf", + "*_p_i_ni_dv_smf", + "*_p_i_sml", + "*_p_i_e_sml", + "*_p_i_e_dv_sml", + "*_p_i_dv_sml", + "*_p_i_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomIntsMissing); + doTestIntFieldFunctionQuery(field); + } + + // no docvalues + for (String r : Arrays.asList("*_p_i_ni", "*_p_i_ni_ns")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", toStringArray(getRandomInts(1, false))); + doTestFieldFunctionQueryError(field, "w/o docValues", toStringArray(getRandomInts(1, false))); + } + + // multivalued, no docvalues + for (String r : + Arrays.asList( + "*_p_i_mv", + "*_p_i_e_mv", + "*_p_i_e_mv_smf", + "*_p_i_e_mv_sml", + "*_p_i_ni_mv", + "*_p_i_ni_ns_mv", + "*_p_i_mv_smf", + "*_p_i_mv_sml")) { + + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", toStringArray(getRandomInts(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldSortError(field, "w/o docValues", toStringArray(getRandomInts(numValues, false))); + doTestFieldFunctionQueryError(field, "multivalued", toStringArray(getRandomInts(1, false))); + doTestFieldFunctionQueryError( + field, "multivalued", toStringArray(getRandomInts(numValues, false))); + } + + // multivalued, w/ docValues + for (String r : + Arrays.asList( + "*_p_i_ni_mv_dv", + "*_p_i_ni_dv_ns_mv", + "*_p_i_e_mv_dv", + "*_p_i_e_mv_dv_smf", + "*_p_i_e_mv_dv_sml", + "*_p_i_dv_ns_mv", + "*_p_i_mv_dv", + "*_p_i_mv_dv_smf", + "*_p_i_ni_mv_dv_smf", + "*_p_i_mv_dv_sml", + "*_p_i_ni_mv_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + + // NOTE: only testing one value per doc here, but TestMinMaxOnMultiValuedField + // covers this in more depth + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomInts); + + // value source (w/o field(...,min|max)) usage should still error... + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldFunctionQueryError(field, "multivalued", toStringArray(getRandomInts(1, false))); + doTestFieldFunctionQueryError( + field, "multivalued", toStringArray(getRandomInts(numValues, false))); + } + + assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); + } + + @Test + public void testIntFieldFacetField() throws Exception { + doTestFieldFacetField("number_p_i", "number_p_i_dv", getSequentialStringArrayWithInts(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField("number_p_i_e", "number_p_i_e_dv", getSequentialStringArrayWithInts(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField("number_p_i", "number_p_i_dv", toStringArray(getRandomInts(10, false))); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_i_e", "number_p_i_e_dv", toStringArray(getRandomInts(10, false))); + } + + @Test + public void testIntFieldRangeFacet() { + String nonDocValuesField = "number_p_i" + (random().nextBoolean() ? "_e" : ""); + String docValuesField = nonDocValuesField + "_dv"; + int numValues = 10 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List sortedValues; + int max; + do { + values = getRandomInts(numValues, false); + sortedValues = values.stream().sorted().collect(Collectors.toList()); + } while ((max = sortedValues.get(sortedValues.size() - 1)) + >= Integer.MAX_VALUE - numValues); // leave room for rounding + int min = sortedValues.get(0); + int gap = (int) (((long) (max + numValues) - (long) min) / (long) numBuckets); + int[] bucketCount = new int[numBuckets]; + int bucketNum = 0; + int minBucketVal = min; + for (Integer value : sortedValues) { + while (((long) value - (long) minBucketVal) >= (long) gap) { + ++bucketNum; + minBucketVal += gap; + } + ++bucketCount[bucketNum]; + } + + for (int i = 0; i < numValues; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + docValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i)))); + } + assertU(commit()); + + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + numValues + "']"; + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap)), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + } + + @Test + public void testIntFieldStats() { + int numValues = 10 * RANDOM_MULTIPLIER; + // don't produce numbers with exponents, since XPath comparison operators can't handle them + List values = getRandomInts(numValues, false, 9999999); + // System.err.println(Arrays.toString(values.toArray(new Integer[values.size()]))); + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + double min = (double) sortedValues.get(0); + double max = (double) sortedValues.get(sortedValues.size() - 1); + + String[] valArray = toStringArray(values); + doTestFieldStats("number_p_i", "number_p_i_dv", valArray, min, max, numValues, 1, 0D); + doTestFieldStats("number_p_i_e", "number_p_i_e_dv", valArray, min, max, numValues, 1, 0D); + doTestFieldStats("number_p_i", "number_p_i_dv", valArray, min, max, numValues, 1, 0D); + doTestFieldStats("number_p_i_e_mv", "number_p_i_e_mv_dv", valArray, min, max, numValues, 1, 0D); + } + + @Test + public void testIntFieldMultiValuedExactQuery() throws Exception { + String[] ints = toStringArray(getRandomInts(20, false)); + doTestFieldMultiValuedExactQuery("number_p_i_mv", ints); + doTestFieldMultiValuedExactQuery("number_p_i_e_mv", ints); + doTestFieldMultiValuedExactQuery("number_p_i_e_mv_dv", ints); + doTestFieldMultiValuedExactQuery("number_p_i_mv_dv", ints); + doTestFieldMultiValuedExactQuery("number_p_i_ni_mv_dv", ints); + } + + @Test + public void testIntFieldMultiValuedNonSearchableExactQuery() throws Exception { + String[] ints = toStringArray(getRandomInts(20, false)); + doTestFieldMultiValuedExactQuery("number_p_i_ni_mv", ints, false); + doTestFieldMultiValuedExactQuery("number_p_i_ni_ns_mv", ints, false); + } + + @Test + public void testIntFieldMultiValuedReturn() throws Exception { + String[] ints = toStringArray(getRandomInts(20, false)); + doTestFieldMultiValuedReturn("number_p_i_mv", "int", ints); + doTestFieldMultiValuedReturn("number_p_i_e_mv", "int", ints); + doTestFieldMultiValuedReturn("number_p_i_ni_mv_dv", "int", ints); + doTestFieldMultiValuedReturn("number_p_i_dv_ns_mv", "int", ints); + } + + @Test + public void testIntFieldMultiValuedRangeQuery() throws Exception { + String[] ints = + toStringArray(getRandomInts(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedRangeQuery("number_p_i_mv", "int", ints); + doTestFieldMultiValuedRangeQuery("number_p_i_e_mv", "int", ints); + doTestFieldMultiValuedRangeQuery("number_p_i_ni_mv_dv", "int", ints); + doTestFieldMultiValuedRangeQuery("number_p_i_mv_dv", "int", ints); + doTestFieldMultiValuedRangeQuery("number_p_i_e_mv_dv", "int", ints); + } + + @Test + public void testIntFieldNotIndexed() throws Exception { + String[] ints = toStringArray(getRandomInts(10, false)); + doTestFieldNotIndexed("number_p_i_ni", ints); + doTestFieldNotIndexed("number_p_i_ni_mv", ints); + } + + // TODO MV SORT? + @Test + public void testIntFieldMultiValuedFacetField() throws Exception { + doTestFieldMultiValuedFacetField( + "number_p_i_mv", "number_p_i_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFacetField( + "number_p_i_e_mv", "number_p_i_e_mv_dv", getSequentialStringArrayWithInts(20)); + String[] randomSortedInts = + toStringArray(getRandomInts(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedFacetField("number_p_i_mv", "number_p_i_mv_dv", randomSortedInts); + doTestFieldMultiValuedFacetField("number_p_i_e_mv", "number_p_i_e_mv_dv", randomSortedInts); + } + + @Test + public void testIntFieldMultiValuedRangeFacet() { + String nonDocValuesField = "number_p_i" + (random().nextBoolean() ? "_e" : "") + "_mv"; + String docValuesField = nonDocValuesField + "_dv"; + int numValues = 20 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List> sortedValues; + int max; + do { + values = getRandomInts(numValues, false); + sortedValues = toAscendingPosVals(values, true); + } while ((max = sortedValues.get(sortedValues.size() - 1).val) + >= Integer.MAX_VALUE - numValues); // leave room for rounding + int min = sortedValues.get(0).val; + int gap = (int) (((long) (max + numValues) - (long) min) / (long) numBuckets); + List> docIdBucket = new ArrayList<>(numBuckets); + for (int i = 0; i < numBuckets; ++i) { + docIdBucket.add(new HashSet<>()); + } + int bucketNum = 0; + int minBucketVal = min; + for (PosVal value : sortedValues) { + while ((long) value.val - (long) minBucketVal >= gap) { + ++bucketNum; + minBucketVal += gap; + } + docIdBucket.get(bucketNum).add(value.pos / 2); // each doc gets two consecutive values + } + for (int i = 0; i < numValues; i += 2) { + assertU( + adoc( + "id", + String.valueOf(i / 2), + docValuesField, + String.valueOf(values.get(i)), + docValuesField, + String.valueOf(values.get(i + 1)), + nonDocValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i + 1)))); + } + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + minBucketVal = min; + testStrings[numBuckets] = "//*[@numFound='" + (numValues / 2) + "']"; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "indent", + "on"), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter", + "indent", + "on"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + } + + @Test + public void testIntFieldMultiValuedFunctionQuery() throws Exception { + doTestFieldMultiValuedFunctionQuery( + "number_p_i_mv", "number_p_i_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_i_e_mv", "number_p_i_e_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_i_mv", + "number_p_i_mv_dv", + toStringArray(getRandomInts(20, false).stream().sorted().collect(Collectors.toList()))); + doTestFieldMultiValuedFunctionQuery( + "number_p_i_e_mv", + "number_p_i_e_mv_dv", + toStringArray(getRandomInts(20, false).stream().sorted().collect(Collectors.toList()))); + } + + @Test + public void testIntFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + doTestIntFieldsAtomicUpdates("number_p_i"); + doTestIntFieldsAtomicUpdates("number_p_i_e"); + doTestIntFieldsAtomicUpdates("number_p_i_dv"); + doTestIntFieldsAtomicUpdates("number_p_i_e_dv"); + doTestIntFieldsAtomicUpdates("number_p_i_dv_ns"); + } + + @Test + public void testMultiValuedIntFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + String[] ints = toStringArray(getRandomInts(3, false)); + doTestMultiValuedFieldsAtomicUpdates("number_p_i_mv", "int", ints); + doTestMultiValuedFieldsAtomicUpdates("number_p_i_e_mv", "int", ints); + doTestMultiValuedFieldsAtomicUpdates("number_p_i_e_mv_dv", "int", ints); + doTestMultiValuedFieldsAtomicUpdates("number_p_i_ni_mv_dv", "int", ints); + doTestMultiValuedFieldsAtomicUpdates("number_p_i_dv_ns_mv", "int", ints); + } + + private String[] toStringArray(List list) { + return list.stream() + .map(String::valueOf) + .collect(Collectors.toList()) + .toArray(new String[list.size()]); + } + + private static class PosVal> { + int pos; + T val; + + PosVal(int pos, T val) { + this.pos = pos; + this.val = val; + } + + @Override + public String toString() { + return "(" + pos + ": " + val.toString() + ")"; + } + } + + /** + * Primary sort by value, with nulls either first or last as specified, and then secondary sort by + * position. + */ + private > Comparator> getPosValComparator( + final boolean ascending, final boolean nullsFirst) { + return (o1, o2) -> { + if (o1.val == null) { + if (o2.val == null) { + return ascending ? Integer.compare(o1.pos, o2.pos) : Integer.compare(o2.pos, o1.pos); + } else { + return nullsFirst ? -1 : 1; + } + } else if (o2.val == null) { + return nullsFirst ? 1 : -1; + } else { + return ascending ? o1.val.compareTo(o2.val) : o2.val.compareTo(o1.val); + } + }; + } + + /** + * Primary ascending sort by value, with missing values (represented as null) either first or last + * as specified, and then secondary ascending sort by position. + */ + private > String[] toAscendingStringArray( + List list, boolean missingFirst) { + return toStringArray( + toAscendingPosVals(list, missingFirst).stream() + .map(pv -> pv.val) + .collect(Collectors.toList())); + } + + /** + * Primary ascending sort by value, with missing values (represented as null) either first or last + * as specified, and then secondary ascending sort by position. + * + * @return a list of the (originally) positioned values sorted as described above. + */ + private > List> toAscendingPosVals( + List list, boolean missingFirst) { + List> posVals = + IntStream.range(0, list.size()) + .mapToObj(i -> new PosVal<>(i, list.get(i))) + .collect(Collectors.toList()); + posVals.sort(getPosValComparator(true, missingFirst)); + return posVals; + } + + /** + * Primary descending sort by value, with missing values (represented as null) either first or + * last as specified, and then secondary descending sort by position. + * + * @return a list of the (originally) positioned values sorted as described above. + */ + private > List> toDescendingPosVals( + List list, boolean missingFirst) { + List> posVals = + IntStream.range(0, list.size()) + .mapToObj(i -> new PosVal<>(i, list.get(i))) + .collect(Collectors.toList()); + posVals.sort(getPosValComparator(false, missingFirst)); + return posVals; + } + + @Test + public void testIntFieldSetQuery() { + doTestSetQueries("number_p_i", toStringArray(getRandomInts(20, false)), false); + doTestSetQueries("number_p_i_dv", toStringArray(getRandomInts(20, false)), false); + doTestSetQueries("number_p_i_mv", toStringArray(getRandomInts(20, false)), true); + doTestSetQueries("number_p_i_mv_dv", toStringArray(getRandomInts(20, false)), true); + doTestSetQueries("number_p_i_ni_dv", toStringArray(getRandomInts(20, false)), false); + doTestSetQueries("number_p_i_e", toStringArray(getRandomInts(20, false)), false); + doTestSetQueries("number_p_i_e_dv", toStringArray(getRandomInts(20, false)), false); + doTestSetQueries("number_p_i_e_mv", toStringArray(getRandomInts(20, false)), true); + doTestSetQueries("number_p_i_e_mv_dv", toStringArray(getRandomInts(20, false)), true); + } + + // DoubleField + + @Test + public void testDoubleFieldExactQuery() throws Exception { + doTestFloatFieldExactQuery("number_p_d", true); + doTestFloatFieldExactQuery("number_p_d_mv", true); + doTestFloatFieldExactQuery("number_p_d_dv", true); + doTestFloatFieldExactQuery("number_p_d_mv_dv", true); + doTestFloatFieldExactQuery("number_p_d_ni_dv", true); + doTestFloatFieldExactQuery("number_p_d_ni_ns_dv", true); + doTestFloatFieldExactQuery("number_p_d_ni_dv_ns", true); + doTestFloatFieldExactQuery("number_p_d_ni_mv_dv", true); + doTestFloatFieldExactQuery("number_p_d_e", true); + doTestFloatFieldExactQuery("number_p_d_e_mv", true); + doTestFloatFieldExactQuery("number_p_d_e_dv", true); + doTestFloatFieldExactQuery("number_p_d_e_mv_dv", true); + } + + @Test + public void testDoubleFieldNonSearchableExactQuery() throws Exception { + doTestFloatFieldExactQuery("number_p_d_ni", false, true); + doTestFloatFieldExactQuery("number_p_d_ni_ns", false, true); + } + + @Test + public void testDoubleFieldReturn() throws Exception { + int numValues = 10 * RANDOM_MULTIPLIER; + String[] doubles = toStringArray(getRandomDoubles(numValues, false)); + doTestFieldReturn("number_p_d", "double", doubles); + doTestFieldReturn("number_p_d_dv_ns", "double", doubles); + } + + @Test + public void testDoubleFieldRangeQuery() throws Exception { + doTestFloatFieldRangeQuery("number_p_d", "double", true); + doTestFloatFieldRangeQuery("number_p_d_ni_ns_dv", "double", true); + doTestFloatFieldRangeQuery("number_p_d_dv", "double", true); + doTestFloatFieldRangeQuery("number_p_d_e", "double", true); + doTestFloatFieldRangeQuery("number_p_d_e_dv", "double", true); + } + + @Test + public void testDoubleFieldNonSearchableRangeQuery() throws Exception { + doTestFieldNonSearchableRangeQuery("number_p_d_ni", toStringArray(getRandomDoubles(1, false))); + doTestFieldNonSearchableRangeQuery( + "number_p_d_ni_ns", toStringArray(getRandomDoubles(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldNonSearchableRangeQuery( + "number_p_d_ni_ns_mv", toStringArray(getRandomDoubles(numValues, false))); + } + + @Test + public void testDoubleFieldSortAndFunction() throws Exception { + final SortedSet regexToTest = dynFieldRegexesForType(DoubleField.class); + final List sequential = + Arrays.asList("0.0", "1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0"); + List randomDoubles = getRandomDoubles(10, false); + List randomDoublesMissing = getRandomDoubles(10, true); + + for (String r : + Arrays.asList( + "*_p_d", + "*_p_d_e", + "*_p_d_e_dv", + "*_p_d_dv", + "*_p_d_dv_ns", + "*_p_d_ni_dv", + "*_p_d_ni_dv_ns", + "*_p_d_ni_ns_dv")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomDoubles); + doTestDoubleFieldFunctionQuery(field); + } + + for (String r : + Arrays.asList( + "*_p_d_smf", + "*_p_d_e_smf", + "*_p_d_e_dv_smf", + "*_p_d_dv_smf", + "*_p_d_ni_dv_smf", + "*_p_d_sml", + "*_p_d_e_sml", + "*_p_d_e_dv_sml", + "*_p_d_dv_sml", + "*_p_d_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomDoublesMissing); + doTestDoubleFieldFunctionQuery(field); + } + + for (String r : Arrays.asList("*_p_d_ni", "*_p_d_ni_ns")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", "42.34"); + doTestFieldFunctionQueryError(field, "w/o docValues", "42.34"); + } + + // multivalued, no docvalues + for (String r : + Arrays.asList( + "*_p_d_mv", + "*_p_d_ni_mv", + "*_p_d_ni_ns_mv", + "*_p_d_mv_smf", + "*_p_d_mv_sml", + "*_p_d_e_mv", + "*_p_d_e_mv_smf", + "*_p_d_e_mv_sml")) { + + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", "42.34"); + doTestFieldSortError(field, "w/o docValues", "42.34", "66.6"); + doTestFieldFunctionQueryError(field, "multivalued", "42.34"); + doTestFieldFunctionQueryError(field, "multivalued", "42.34", "66.6"); + } + + // multivalued, w/ docValues + for (String r : + Arrays.asList( + "*_p_d_ni_mv_dv", + "*_p_d_ni_dv_ns_mv", + "*_p_d_dv_ns_mv", + "*_p_d_mv_dv", + "*_p_d_e_mv_dv", + "*_p_d_e_mv_dv_smf", + "*_p_d_e_mv_dv_sml", + "*_p_d_mv_dv_smf", + "*_p_d_ni_mv_dv_smf", + "*_p_d_mv_dv_sml", + "*_p_d_ni_mv_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + + // NOTE: only testing one value per doc here, but TestMinMaxOnMultiValuedField + // covers this in more depth + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomDoubles); + + // value source (w/o field(...,min|max)) usuage should still error... + doTestFieldFunctionQueryError(field, "multivalued", "42.34"); + doTestFieldFunctionQueryError(field, "multivalued", "42.34", "66.6"); + } + assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); + } + + @Test + public void testDoubleFieldFacetField() throws Exception { + doTestFieldFacetField("number_p_d", "number_p_d_dv", getSequentialStringArrayWithDoubles(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_d_e", "number_p_d_e_dv", getSequentialStringArrayWithDoubles(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_d", "number_p_d_dv", toStringArray(getRandomDoubles(10, false))); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_d_e", "number_p_d_e_dv", toStringArray(getRandomDoubles(10, false))); + } + + @Test + public void testDoubleFieldRangeFacet() { + String nonDocValuesField = "number_p_d" + (random().nextBoolean() ? "_e" : ""); + String docValuesField = nonDocValuesField + "_dv"; + int numValues = 10 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values, sortedValues; + double min, max, gap, buffer; + do { + values = getRandomDoubles(numValues, false); + sortedValues = values.stream().sorted().collect(Collectors.toList()); + min = sortedValues.get(0); + max = sortedValues.get(sortedValues.size() - 1); + buffer = + BigDecimal.valueOf(max) + .subtract(BigDecimal.valueOf(min)) + .divide(BigDecimal.valueOf(numValues / 2), RoundingMode.HALF_UP) + .doubleValue(); + gap = + BigDecimal.valueOf(max) + .subtract(BigDecimal.valueOf(min)) + .add(BigDecimal.valueOf(buffer * 2.0D)) + .divide(BigDecimal.valueOf(numBuckets), RoundingMode.HALF_UP) + .doubleValue(); + } while (max >= Double.MAX_VALUE - buffer || min <= -Double.MAX_VALUE + buffer); + // System.err.println("min: " + min + " max: " + max + " gap: " + gap + " buffer: " + + // buffer); + int[] bucketCount = new int[numBuckets]; + int bucketNum = 0; + double minBucketVal = min - buffer; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + for (double value : sortedValues) { + // System.err.println("value: " + value); + while (value - minBucketVal >= gap) { + ++bucketNum; + minBucketVal += gap; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + } + ++bucketCount[bucketNum]; + } + + for (int i = 0; i < numValues; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + docValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i)))); + } + assertU(commit()); + + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + numValues + "']"; + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap)), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + } + + @Test + public void testDoubleFieldStats() { + int numValues = 10 * RANDOM_MULTIPLIER; + // don't produce numbers with exponents, since XPath comparison operators can't handle them: 7 + // digits of precision + List values = + getRandomInts(numValues, false, 9999999).stream() + .map(v -> (float) ((double) v * Math.pow(10D, -1 * random().nextInt(8)))) + .collect(Collectors.toList()); + // System.err.println(Arrays.toString(values.toArray(new Float[values.size()]))); + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + double min = (double) sortedValues.get(0); + double max = (double) sortedValues.get(sortedValues.size() - 1); + + String[] valArray = toStringArray(values); + doTestFieldStats("number_p_d", "number_p_d_dv", valArray, min, max, numValues, 1, 1E-7D); + doTestFieldStats("number_p_d_e", "number_p_d_e_dv", valArray, min, max, numValues, 1, 1E-7D); + doTestFieldStats("number_p_d_mv", "number_p_d_mv_dv", valArray, min, max, numValues, 1, 1E-7D); + doTestFieldStats( + "number_p_d_e_mv", "number_p_d_e_mv_dv", valArray, min, max, numValues, 1, 1E-7D); + } + + @Test + public void testDoubleFieldMultiValuedExactQuery() throws Exception { + String[] doubles = toStringArray(getRandomDoubles(20, false)); + doTestFieldMultiValuedExactQuery("number_p_d_mv", doubles); + doTestFieldMultiValuedExactQuery("number_p_d_e_mv", doubles); + doTestFieldMultiValuedExactQuery("number_p_d_e_mv_dv", doubles); + doTestFieldMultiValuedExactQuery("number_p_d_mv_dv", doubles); + doTestFieldMultiValuedExactQuery("number_p_d_ni_mv_dv", doubles); + } + + @Test + public void testDoubleFieldMultiValuedNonSearchableExactQuery() throws Exception { + String[] doubles = toStringArray(getRandomDoubles(20, false)); + doTestFieldMultiValuedExactQuery("number_p_d_ni_mv", doubles, false); + doTestFieldMultiValuedExactQuery("number_p_d_ni_ns_mv", doubles, false); + } + + @Test + public void testDoubleFieldMultiValuedReturn() throws Exception { + String[] doubles = toStringArray(getRandomDoubles(20, false)); + doTestFieldMultiValuedReturn("number_p_d_mv", "double", doubles); + doTestFieldMultiValuedReturn("number_p_d_e_mv", "double", doubles); + doTestFieldMultiValuedReturn("number_p_d_ni_mv_dv", "double", doubles); + doTestFieldMultiValuedReturn("number_p_d_dv_ns_mv", "double", doubles); + } + + @Test + public void testDoubleFieldMultiValuedRangeQuery() throws Exception { + String[] doubles = + toStringArray(getRandomDoubles(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedRangeQuery("number_p_d_mv", "double", doubles); + doTestFieldMultiValuedRangeQuery("number_p_d_e_mv", "double", doubles); + doTestFieldMultiValuedRangeQuery("number_p_d_e_mv_dv", "double", doubles); + doTestFieldMultiValuedRangeQuery("number_p_d_ni_mv_dv", "double", doubles); + doTestFieldMultiValuedRangeQuery("number_p_d_mv_dv", "double", doubles); + } + + @Test + public void testDoubleFieldMultiValuedFacetField() throws Exception { + doTestFieldMultiValuedFacetField( + "number_p_d_mv", "number_p_d_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFacetField( + "number_p_d_e_mv", "number_p_d_e_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFacetField( + "number_p_d_mv", "number_p_d_mv_dv", toStringArray(getRandomDoubles(20, false))); + doTestFieldMultiValuedFacetField( + "number_p_d_e_mv", "number_p_d_e_mv_dv", toStringArray(getRandomDoubles(20, false))); + } + + @Test + public void testDoubleFieldMultiValuedRangeFacet() { + String nonDocValuesField = "number_p_d" + (random().nextBoolean() ? "_e" : "") + "_mv"; + String docValuesField = nonDocValuesField + "_dv"; + SchemaField dvSchemaField = h.getCore().getLatestSchema().getField(docValuesField); + assertTrue(dvSchemaField.multiValued()); + assertTrue(dvSchemaField.hasDocValues()); + assertTrue(dvSchemaField.getType() instanceof NumericField); + + SchemaField nonDvSchemaField = h.getCore().getLatestSchema().getField(nonDocValuesField); + assertTrue(nonDvSchemaField.multiValued()); + assertFalse(nonDvSchemaField.hasDocValues()); + assertTrue(nonDvSchemaField.getType() instanceof NumericField); + + int numValues = 20 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List> sortedValues; + double min, max, gap, buffer; + do { + values = getRandomDoubles(numValues, false); + sortedValues = toAscendingPosVals(values, true); + min = sortedValues.get(0).val; + max = sortedValues.get(sortedValues.size() - 1).val; + buffer = + BigDecimal.valueOf(max) + .subtract(BigDecimal.valueOf(min)) + .divide(BigDecimal.valueOf(numValues / 2), RoundingMode.HALF_UP) + .doubleValue(); + gap = + BigDecimal.valueOf(max) + .subtract(BigDecimal.valueOf(min)) + .add(BigDecimal.valueOf(buffer * 2.0D)) + .divide(BigDecimal.valueOf(numBuckets), RoundingMode.HALF_UP) + .doubleValue(); + } while (max >= Double.MAX_VALUE - buffer || min <= -Double.MAX_VALUE + buffer); + // System.err.println("min: " + min + " max: " + max + " gap: " + gap + " buffer: " + + // buffer); + List> docIdBucket = new ArrayList<>(numBuckets); + for (int i = 0; i < numBuckets; ++i) { + docIdBucket.add(new HashSet<>()); + } + int bucketNum = 0; + double minBucketVal = min - buffer; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + for (PosVal value : sortedValues) { + // System.err.println("value.val: " + value.val); + while (value.val - minBucketVal >= gap) { + ++bucketNum; + minBucketVal += gap; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + } + docIdBucket.get(bucketNum).add(value.pos / 2); // each doc gets two consecutive values + } + for (int i = 0; i < numValues; i += 2) { + assertU( + adoc( + "id", + String.valueOf(i / 2), + docValuesField, + String.valueOf(values.get(i)), + docValuesField, + String.valueOf(values.get(i + 1)), + nonDocValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i + 1)))); + } + assertU(commit()); + + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + (numValues / 2) + "']"; + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "indent", + "on"), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter", + "indent", + "on"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + } + + @Test + public void testDoubleFieldMultiValuedFunctionQuery() throws Exception { + doTestFieldMultiValuedFunctionQuery( + "number_p_d_mv", "number_p_d_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_d_e_mv", "number_p_d_e_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_d_mv", + "number_p_d_mv_dv", + toAscendingStringArray(getRandomFloats(20, false), true)); + doTestFieldMultiValuedFunctionQuery( + "number_p_d_e_mv", + "number_p_d_e_mv_dv", + toAscendingStringArray(getRandomFloats(20, false), true)); + } + + @Test + public void testDoubleFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + doTestDoubleFieldsAtomicUpdates("number_p_d"); + doTestDoubleFieldsAtomicUpdates("number_p_d_e"); + doTestDoubleFieldsAtomicUpdates("number_p_d_dv"); + doTestDoubleFieldsAtomicUpdates("number_p_d_e_dv"); + doTestDoubleFieldsAtomicUpdates("number_p_d_dv_ns"); + } + + @Test + public void testMultiValuedDoubleFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + String[] doubles = toStringArray(getRandomDoubles(3, false)); + doTestMultiValuedFieldsAtomicUpdates("number_p_d_mv", "double", doubles); + doTestMultiValuedFieldsAtomicUpdates("number_p_d_e_mv", "double", doubles); + doTestMultiValuedFieldsAtomicUpdates("number_p_d_e_mv_dv", "double", doubles); + doTestMultiValuedFieldsAtomicUpdates("number_p_d_mv_dv", "double", doubles); + doTestMultiValuedFieldsAtomicUpdates("number_p_d_ni_mv_dv", "double", doubles); + doTestMultiValuedFieldsAtomicUpdates("number_p_d_dv_ns_mv", "double", doubles); + } + + @Test + public void testDoubleFieldNotIndexed() throws Exception { + String[] doubles = toStringArray(getRandomDoubles(10, false)); + doTestFieldNotIndexed("number_p_d_ni", doubles); + doTestFieldNotIndexed("number_p_d_ni_mv", doubles); + } + + private void doTestFloatFieldsAtomicUpdates(String field) { + float number1 = getRandomFloats(1, false).get(0); + float number2; + double inc1; + for (; ; ) { + number2 = getRandomFloats(1, false).get(0); + inc1 = (double) number2 - (double) number1; + if (Math.abs(inc1) < (double) Float.MAX_VALUE) { + number2 = number1 + (float) inc1; + break; + } + } + assertU(adoc(sdoc("id", "1", field, String.valueOf(number1)))); + assertU(commit()); + + assertU(adoc(sdoc("id", "1", field, Map.of("inc", (float) inc1)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/float[@name='" + field + "'][.='" + number2 + "']"); + + float number3 = getRandomFloats(1, false).get(0); + assertU(adoc(sdoc("id", "1", field, Map.of("set", number3)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/float[@name='" + field + "'][.='" + number3 + "']"); + } + + private void doTestDoubleFieldsAtomicUpdates(String field) { + double number1 = getRandomDoubles(1, false).get(0); + double number2; + BigDecimal inc1; + for (; ; ) { + number2 = getRandomDoubles(1, false).get(0); + inc1 = BigDecimal.valueOf(number2).subtract(BigDecimal.valueOf(number1)); + if (inc1.abs().compareTo(BigDecimal.valueOf(Double.MAX_VALUE)) <= 0) { + number2 = number1 + inc1.doubleValue(); + break; + } + } + assertU(adoc(sdoc("id", "1", field, String.valueOf(number1)))); + assertU(commit()); + + assertU(adoc(sdoc("id", "1", field, Map.of("inc", inc1.doubleValue())))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/double[@name='" + field + "'][.='" + number2 + "']"); + + double number3 = getRandomDoubles(1, false).get(0); + assertU(adoc(sdoc("id", "1", field, Map.of("set", number3)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/double[@name='" + field + "'][.='" + number3 + "']"); + } + + @Test + public void testDoubleFieldSetQuery() { + doTestSetQueries("number_p_d", toStringArray(getRandomDoubles(20, false)), false); + doTestSetQueries("number_p_d_dv", toStringArray(getRandomDoubles(20, false)), false); + doTestSetQueries("number_p_d_mv", toStringArray(getRandomDoubles(20, false)), true); + doTestSetQueries("number_p_d_mv_dv", toStringArray(getRandomDoubles(20, false)), true); + doTestSetQueries("number_p_d_ni_dv", toStringArray(getRandomDoubles(20, false)), false); + doTestSetQueries("number_p_d_e", toStringArray(getRandomDoubles(20, false)), false); + doTestSetQueries("number_p_d_e_dv", toStringArray(getRandomDoubles(20, false)), false); + doTestSetQueries("number_p_d_e_mv", toStringArray(getRandomDoubles(20, false)), true); + doTestSetQueries("number_p_d_e_mv_dv", toStringArray(getRandomDoubles(20, false)), true); + } + + // Float + + @Test + public void testFloatFieldExactQuery() throws Exception { + doTestFloatFieldExactQuery("number_p_f", false); + doTestFloatFieldExactQuery("number_p_f_mv", false); + doTestFloatFieldExactQuery("number_p_f_dv", false); + doTestFloatFieldExactQuery("number_p_f_mv_dv", false); + doTestFloatFieldExactQuery("number_p_f_ni_dv", false); + doTestFloatFieldExactQuery("number_p_f_ni_ns_dv", false); + doTestFloatFieldExactQuery("number_p_f_ni_dv_ns", false); + doTestFloatFieldExactQuery("number_p_f_ni_mv_dv", false); + doTestFloatFieldExactQuery("number_p_f_e", false); + doTestFloatFieldExactQuery("number_p_f_e_mv", false); + doTestFloatFieldExactQuery("number_p_f_e_dv", false); + doTestFloatFieldExactQuery("number_p_f_e_mv_dv", false); + } + + @Test + public void testFloatFieldNonSearchableExactQuery() throws Exception { + doTestFloatFieldExactQuery("number_p_f_ni", false, false); + doTestFloatFieldExactQuery("number_p_f_ni_ns", false, false); + } + + @Test + public void testFloatFieldReturn() throws Exception { + int numValues = 10 * RANDOM_MULTIPLIER; + String[] floats = toStringArray(getRandomFloats(numValues, false)); + doTestFieldReturn("number_p_f", "float", floats); + doTestFieldReturn("number_p_f_e", "float", floats); + doTestFieldReturn("number_p_f_dv", "float", floats); + doTestFieldReturn("number_p_f_e_dv", "float", floats); + doTestFieldReturn("number_p_f_dv_ns", "float", floats); + } + + @Test + public void testFloatFieldRangeQuery() throws Exception { + doTestFloatFieldRangeQuery("number_p_f", "float", false); + doTestFloatFieldRangeQuery("number_p_f_e", "float", false); + doTestFloatFieldRangeQuery("number_p_f_e_dv", "float", false); + doTestFloatFieldRangeQuery("number_p_f_ni_ns_dv", "float", false); + doTestFloatFieldRangeQuery("number_p_f_dv", "float", false); + } + + @Test + public void testFloatFieldNonSearchableRangeQuery() throws Exception { + doTestFieldNonSearchableRangeQuery("number_p_f_ni", toStringArray(getRandomFloats(1, false))); + doTestFieldNonSearchableRangeQuery( + "number_p_f_ni_ns", toStringArray(getRandomFloats(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldNonSearchableRangeQuery( + "number_p_f_ni_ns_mv", toStringArray(getRandomFloats(numValues, false))); + } + + @Test + public void testFloatFieldSortAndFunction() throws Exception { + final SortedSet regexToTest = dynFieldRegexesForType(FloatField.class); + final List sequential = + Arrays.asList("0.0", "1.0", "2.0", "3.0", "4.0", "5.0", "6.0", "7.0", "8.0", "9.0"); + final List randomFloats = getRandomFloats(10, false); + final List randomFloatsMissing = getRandomFloats(10, true); + + for (String r : + Arrays.asList( + "*_p_f", + "*_p_f_e", + "*_p_f_dv", + "*_p_f_e_dv", + "*_p_f_dv_ns", + "*_p_f_ni_dv", + "*_p_f_ni_dv_ns", + "*_p_f_ni_ns_dv")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomFloats); + + doTestFloatFieldFunctionQuery(field); + } + for (String r : + Arrays.asList( + "*_p_f_smf", + "*_p_f_e_smf", + "*_p_f_dv_smf", + "*_p_f_e_dv_smf", + "*_p_f_ni_dv_smf", + "*_p_f_sml", + "*_p_f_e_sml", + "*_p_f_dv_sml", + "*_p_f_e_dv_sml", + "*_p_f_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomFloatsMissing); + doTestFloatFieldFunctionQuery(field); + } + + for (String r : Arrays.asList("*_p_f_ni", "*_p_f_ni_ns")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", "42.34"); + doTestFieldFunctionQueryError(field, "w/o docValues", "42.34"); + } + + // multivalued, no docvalues + for (String r : + Arrays.asList( + "*_p_f_mv", + "*_p_f_e_mv", + "*_p_f_ni_mv", + "*_p_f_ni_ns_mv", + "*_p_f_mv_smf", + "*_p_f_e_mv_smf", + "*_p_f_mv_sml", + "*_p_f_e_mv_sml")) { + + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", "42.34"); + doTestFieldSortError(field, "w/o docValues", "42.34", "66.6"); + doTestFieldFunctionQueryError(field, "multivalued", "42.34"); + doTestFieldFunctionQueryError(field, "multivalued", "42.34", "66.6"); + } + + // multivalued, w/ docValues + for (String r : + Arrays.asList( + "*_p_f_ni_mv_dv", + "*_p_f_ni_dv_ns_mv", + "*_p_f_dv_ns_mv", + "*_p_f_mv_dv", + "*_p_f_e_mv_dv", + "*_p_f_e_mv_dv_smf", + "*_p_f_e_mv_dv_sml", + "*_p_f_mv_dv_smf", + "*_p_f_ni_mv_dv_smf", + "*_p_f_mv_dv_sml", + "*_p_f_ni_mv_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + + // NOTE: only testing one value per doc here, but TestMinMaxOnMultiValuedField + // covers this in more depth + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomFloats); + + // value source (w/o field(...,min|max)) usage should still error... + doTestFieldFunctionQueryError(field, "multivalued", "42.34"); + doTestFieldFunctionQueryError(field, "multivalued", "42.34", "66.6"); + } + assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); + } + + @Test + public void testFloatFieldFacetField() throws Exception { + doTestFieldFacetField("number_p_f", "number_p_f_dv", getSequentialStringArrayWithDoubles(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_f_e", "number_p_f_e_dv", getSequentialStringArrayWithDoubles(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField("number_p_f", "number_p_f_dv", toStringArray(getRandomFloats(10, false))); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_f_e", "number_p_f_e_dv", toStringArray(getRandomFloats(10, false))); + } + + @Test + public void testFloatFieldRangeFacet() { + String nonDocValuesField = "number_p_f" + (random().nextBoolean() ? "_e" : ""); + String docValuesField = nonDocValuesField + "_dv"; + int numValues = 10 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values, sortedValues; + float min, max, gap, buffer; + do { + values = getRandomFloats(numValues, false); + sortedValues = values.stream().sorted().collect(Collectors.toList()); + min = sortedValues.get(0); + max = sortedValues.get(sortedValues.size() - 1); + buffer = (float) (((double) max - (double) min) / (double) numValues / 2.0D); + gap = + (float) + (((double) max + (double) buffer - (double) min + (double) buffer) + / (double) numBuckets); + } while (max >= Float.MAX_VALUE - buffer || min <= -Float.MAX_VALUE + buffer); + // System.err.println("min: " + min + " max: " + max + " gap: " + gap + " buffer: " + + // buffer); + int[] bucketCount = new int[numBuckets]; + int bucketNum = 0; + float minBucketVal = min - buffer; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + for (float value : sortedValues) { + // System.err.println("value: " + value); + while (value - minBucketVal >= gap) { + ++bucketNum; + minBucketVal += gap; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + } + ++bucketCount[bucketNum]; + } + + for (int i = 0; i < numValues; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + docValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i)))); + } + assertU(commit()); + + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + numValues + "']"; + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap)), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + } + + @Test + public void testFloatFieldStats() { + int numValues = 10 * RANDOM_MULTIPLIER; + // don't produce numbers with exponents, since XPath comparison operators can't handle them: 7 + // digits of precision + List values = + getRandomInts(numValues, false, 9999999).stream() + .map(v -> (float) ((double) v * Math.pow(10D, -1 * random().nextInt(8)))) + .collect(Collectors.toList()); + // System.err.println(Arrays.toString(values.toArray(new Float[values.size()]))); + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + double min = (double) sortedValues.get(0); + double max = (double) sortedValues.get(sortedValues.size() - 1); + + String[] valArray = toStringArray(values); + doTestFieldStats("number_p_f", "number_p_f_dv", valArray, min, max, numValues, 1, 1E-7D); + doTestFieldStats("number_p_f_e", "number_p_f_e_dv", valArray, min, max, numValues, 1, 1E-7D); + doTestFieldStats("number_p_f_mv", "number_p_f_mv_dv", valArray, min, max, numValues, 1, 1E-7D); + doTestFieldStats( + "number_p_f_e_mv", "number_p_f_e_mv_dv", valArray, min, max, numValues, 1, 1E-7D); + } + + @Test + public void testFloatFieldMultiValuedExactQuery() throws Exception { + String[] floats = toStringArray(getRandomFloats(20, false)); + doTestFieldMultiValuedExactQuery("number_p_f_mv", floats); + doTestFieldMultiValuedExactQuery("number_p_f_e_mv", floats); + doTestFieldMultiValuedExactQuery("number_p_f_mv_dv", floats); + doTestFieldMultiValuedExactQuery("number_p_f_e_mv_dv", floats); + doTestFieldMultiValuedExactQuery("number_p_f_ni_mv_dv", floats); + } + + @Test + public void testFloatFieldMultiValuedNonSearchableExactQuery() throws Exception { + String[] floats = toStringArray(getRandomFloats(20, false)); + doTestFieldMultiValuedExactQuery("number_p_f_ni_mv", floats, false); + doTestFieldMultiValuedExactQuery("number_p_f_ni_ns_mv", floats, false); + } + + @Test + public void testFloatFieldMultiValuedReturn() throws Exception { + String[] floats = toStringArray(getRandomFloats(20, false)); + doTestFieldMultiValuedReturn("number_p_f_mv", "float", floats); + doTestFieldMultiValuedReturn("number_p_f_e_mv", "float", floats); + doTestFieldMultiValuedReturn("number_p_f_mv_dv", "float", floats); + doTestFieldMultiValuedReturn("number_p_f_e_mv_dv", "float", floats); + doTestFieldMultiValuedReturn("number_p_f_ni_mv_dv", "float", floats); + doTestFieldMultiValuedReturn("number_p_f_dv_ns_mv", "float", floats); + } + + @Test + public void testFloatFieldMultiValuedRangeQuery() throws Exception { + String[] floats = + toStringArray(getRandomFloats(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedRangeQuery("number_p_f_mv", "float", floats); + doTestFieldMultiValuedRangeQuery("number_p_f_e_mv", "float", floats); + doTestFieldMultiValuedRangeQuery("number_p_f_mv_dv", "float", floats); + doTestFieldMultiValuedRangeQuery("number_p_f_e_mv_dv", "float", floats); + doTestFieldMultiValuedRangeQuery("number_p_f_ni_mv_dv", "float", floats); + doTestFieldMultiValuedRangeQuery("number_p_f_mv_dv", "float", floats); + } + + @Test + public void testFloatFieldMultiValuedRangeFacet() { + String nonDocValuesField = "number_p_f" + (random().nextBoolean() ? "_e" : "") + "_mv"; + String docValuesField = nonDocValuesField + "_dv"; + SchemaField dvSchemaField = h.getCore().getLatestSchema().getField(docValuesField); + assertTrue(dvSchemaField.multiValued()); + assertTrue(dvSchemaField.hasDocValues()); + assertTrue(dvSchemaField.getType() instanceof NumericField); + + SchemaField nonDvSchemaField = h.getCore().getLatestSchema().getField(nonDocValuesField); + assertTrue(nonDvSchemaField.multiValued()); + assertFalse(nonDvSchemaField.hasDocValues()); + assertTrue(nonDvSchemaField.getType() instanceof NumericField); + + int numValues = 20 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List> sortedValues; + float min, max, gap, buffer; + do { + values = getRandomFloats(numValues, false); + sortedValues = toAscendingPosVals(values, true); + min = sortedValues.get(0).val; + max = sortedValues.get(sortedValues.size() - 1).val; + buffer = (float) (((double) max - (double) min) / (double) numValues / 2.0D); + gap = + (float) + (((double) max + (double) buffer - (double) min + (double) buffer) + / (double) numBuckets); + } while (max >= Float.MAX_VALUE - buffer || min <= -Float.MAX_VALUE + buffer); + // System.err.println("min: " + min + " max: " + max + " gap: " + gap + " buffer: " + + // buffer); + List> docIdBucket = new ArrayList<>(numBuckets); + for (int i = 0; i < numBuckets; ++i) { + docIdBucket.add(new HashSet<>()); + } + int bucketNum = 0; + float minBucketVal = min - buffer; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + for (PosVal value : sortedValues) { + // System.err.println("value.val: " + value.val); + while (value.val - minBucketVal >= gap) { + ++bucketNum; + minBucketVal += gap; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + } + docIdBucket.get(bucketNum).add(value.pos / 2); // each doc gets two consecutive values + } + for (int i = 0; i < numValues; i += 2) { + assertU( + adoc( + "id", + String.valueOf(i / 2), + docValuesField, + String.valueOf(values.get(i)), + docValuesField, + String.valueOf(values.get(i + 1)), + nonDocValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i + 1)))); + } + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + minBucketVal = min - buffer; + testStrings[numBuckets] = "//*[@numFound='" + (numValues / 2) + "']"; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "indent", + "on"), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min - buffer; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter", + "indent", + "on"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min - buffer), + "facet.range.end", + String.valueOf(max + buffer), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + } + + @Test + public void testFloatFieldMultiValuedFacetField() throws Exception { + doTestFieldMultiValuedFacetField( + "number_p_f_mv", "number_p_f_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFacetField( + "number_p_f_e_mv", "number_p_f_e_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFacetField( + "number_p_f_mv", "number_p_f_mv_dv", toStringArray(getRandomFloats(20, false))); + doTestFieldMultiValuedFacetField( + "number_p_f_e_mv", "number_p_f_e_mv_dv", toStringArray(getRandomFloats(20, false))); + } + + @Test + public void testFloatFieldMultiValuedFunctionQuery() throws Exception { + doTestFieldMultiValuedFunctionQuery( + "number_p_f_mv", "number_p_f_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_f_e_mv", "number_p_f_e_mv_dv", getSequentialStringArrayWithDoubles(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_f_mv", + "number_p_f_mv_dv", + toAscendingStringArray(getRandomFloats(20, false), true)); + doTestFieldMultiValuedFunctionQuery( + "number_p_f_e_mv", + "number_p_f_e_mv_dv", + toAscendingStringArray(getRandomFloats(20, false), true)); + } + + @Test + public void testFloatFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + doTestFloatFieldsAtomicUpdates("number_p_f"); + doTestFloatFieldsAtomicUpdates("number_p_f_e"); + doTestFloatFieldsAtomicUpdates("number_p_f_dv"); + doTestFloatFieldsAtomicUpdates("number_p_f_e_dv"); + doTestFloatFieldsAtomicUpdates("number_p_f_dv_ns"); + } + + @Test + public void testMultiValuedFloatFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + String[] floats = toStringArray(getRandomFloats(3, false)); + doTestMultiValuedFieldsAtomicUpdates("number_p_f_mv", "float", floats); + doTestMultiValuedFieldsAtomicUpdates("number_p_f_e_mv", "float", floats); + doTestMultiValuedFieldsAtomicUpdates("number_p_f_mv_dv", "float", floats); + doTestMultiValuedFieldsAtomicUpdates("number_p_f_e_mv_dv", "float", floats); + doTestMultiValuedFieldsAtomicUpdates("number_p_f_ni_mv_dv", "float", floats); + doTestMultiValuedFieldsAtomicUpdates("number_p_f_dv_ns_mv", "float", floats); + } + + @Test + public void testFloatFieldSetQuery() { + doTestSetQueries("number_p_f", toStringArray(getRandomFloats(20, false)), false); + doTestSetQueries("number_p_f_dv", toStringArray(getRandomFloats(20, false)), false); + doTestSetQueries("number_p_f_mv", toStringArray(getRandomFloats(20, false)), true); + doTestSetQueries("number_p_f_mv_dv", toStringArray(getRandomFloats(20, false)), true); + doTestSetQueries("number_p_f_ni_dv", toStringArray(getRandomFloats(20, false)), false); + doTestSetQueries("number_p_f_ni_mv_dv", toStringArray(getRandomFloats(20, false)), true); + doTestSetQueries("number_p_f_e", toStringArray(getRandomFloats(20, false)), false); + doTestSetQueries("number_p_f_e_dv", toStringArray(getRandomFloats(20, false)), false); + doTestSetQueries("number_p_f_e_mv", toStringArray(getRandomFloats(20, false)), true); + doTestSetQueries("number_p_f_e_mv_dv", toStringArray(getRandomFloats(20, false)), true); + } + + @Test + public void testFloatFieldNotIndexed() throws Exception { + String[] floats = toStringArray(getRandomFloats(10, false)); + doTestFieldNotIndexed("number_p_f_ni", floats); + doTestFieldNotIndexed("number_p_f_ni_mv", floats); + } + + // Long + + @Test + public void testLongFieldExactQuery() throws Exception { + doTestIntFieldExactQuery("number_p_l", true); + doTestIntFieldExactQuery("number_p_l_mv", true); + doTestIntFieldExactQuery("number_p_l_dv", true); + doTestIntFieldExactQuery("number_p_l_mv_dv", true); + doTestIntFieldExactQuery("number_p_l_ni_dv", true); + doTestIntFieldExactQuery("number_p_l_ni_ns_dv", true); + doTestIntFieldExactQuery("number_p_l_ni_dv_ns", true); + doTestIntFieldExactQuery("number_p_l_ni_mv_dv", true); + doTestIntFieldExactQuery("number_p_l_e", true); + doTestIntFieldExactQuery("number_p_l_e_mv", true); + doTestIntFieldExactQuery("number_p_l_e_dv", true); + doTestIntFieldExactQuery("number_p_l_e_mv_dv", true); + } + + @Test + public void testLongFieldNonSearchableExactQuery() throws Exception { + doTestIntFieldExactQuery("number_p_l_ni", true, false); + doTestIntFieldExactQuery("number_p_l_ni_ns", true, false); + } + + @Test + public void testLongFieldReturn() throws Exception { + int numValues = 10 * RANDOM_MULTIPLIER; + String[] longs = toStringArray(getRandomLongs(numValues, false)); + doTestFieldReturn("number_p_l", "long", longs); + doTestFieldReturn("number_p_l_e", "long", longs); + doTestFieldReturn("number_p_l_dv", "long", longs); + doTestFieldReturn("number_p_l_e_dv", "long", longs); + doTestFieldReturn("number_p_l_dv_ns", "long", longs); + } + + @Test + public void testLongFieldRangeQuery() throws Exception { + doTestIntFieldRangeQuery("number_p_l", "long", true); + doTestIntFieldRangeQuery("number_p_l_ni_ns_dv", "long", true); + doTestIntFieldRangeQuery("number_p_l_dv", "long", true); + } + + @Test + public void testLongFieldNonSearchableRangeQuery() throws Exception { + doTestFieldNonSearchableRangeQuery("number_p_l_ni", toStringArray(getRandomLongs(1, false))); + doTestFieldNonSearchableRangeQuery("number_p_l_ni_ns", toStringArray(getRandomLongs(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldNonSearchableRangeQuery( + "number_p_l_ni_ns_mv", toStringArray(getRandomLongs(numValues, false))); + } + + @Test + public void testLongFieldSortAndFunction() throws Exception { + final SortedSet regexToTest = dynFieldRegexesForType(LongField.class); + final List vals = + Arrays.asList( + (long) Integer.MIN_VALUE, + 1L, + 2L, + 3L, + 4L, + 5L, + 6L, + 7L, + (long) Integer.MAX_VALUE, + Long.MAX_VALUE); + final List randomLongs = getRandomLongs(10, false); + final List randomLongsMissing = getRandomLongs(10, true); + + for (String r : + Arrays.asList( + "*_p_l", + "*_p_l_e", + "*_p_l_dv", + "*_p_l_e_dv", + "*_p_l_dv_ns", + "*_p_l_ni_dv", + "*_p_l_ni_dv_ns", + "*_p_l_ni_ns_dv")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, vals); + doTestFieldSort(field, randomLongs); + doTestLongFieldFunctionQuery(field); + } + + for (String r : + Arrays.asList( + "*_p_l_smf", + "*_p_l_e_smf", + "*_p_l_dv_smf", + "*_p_l_e_dv_smf", + "*_p_l_ni_dv_smf", + "*_p_l_sml", + "*_p_l_e_sml", + "*_p_l_dv_sml", + "*_p_l_e_dv_sml", + "*_p_l_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, vals); + doTestFieldSort(field, randomLongsMissing); + doTestLongFieldFunctionQuery(field); + } + + // no docvalues + for (String r : Arrays.asList("*_p_l_ni", "*_p_l_ni_ns")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", toStringArray(getRandomLongs(1, false))); + doTestFieldFunctionQueryError( + field, "w/o docValues", toStringArray(getRandomLongs(1, false))); + } + + // multivalued, no docvalues + for (String r : + Arrays.asList( + "*_p_l_mv", + "*_p_l_e_mv", + "*_p_l_ni_mv", + "*_p_l_ni_ns_mv", + "*_p_l_mv_smf", + "*_p_l_e_mv_smf", + "*_p_l_mv_sml", + "*_p_l_e_mv_sml")) { + + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", toStringArray(getRandomLongs(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldSortError(field, "w/o docValues", toStringArray(getRandomLongs(numValues, false))); + doTestFieldFunctionQueryError(field, "multivalued", toStringArray(getRandomLongs(1, false))); + doTestFieldFunctionQueryError( + field, "multivalued", toStringArray(getRandomLongs(numValues, false))); + } + // multivalued, w/ docValues + for (String r : + Arrays.asList( + "*_p_l_ni_mv_dv", + "*_p_l_ni_dv_ns_mv", + "*_p_l_dv_ns_mv", + "*_p_l_mv_dv", + "*_p_l_e_mv_dv", + "*_p_l_e_mv_dv_smf", + "*_p_l_e_mv_dv_sml", + "*_p_l_mv_dv_smf", + "*_p_l_ni_mv_dv_smf", + "*_p_l_mv_dv_sml", + "*_p_l_ni_mv_dv_sml")) { + + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + + // NOTE: only testing one value per doc here, but TestMinMaxOnMultiValuedField + // covers this in more depth + doTestFieldSort(field, vals); + doTestFieldSort(field, randomLongs); + + // value source (w/o field(...,min|max)) usage should still error... + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldFunctionQueryError(field, "multivalued", toStringArray(getRandomLongs(1, false))); + doTestFieldFunctionQueryError( + field, "multivalued", toStringArray(getRandomLongs(numValues, false))); + } + assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); + } + + @Test + public void testLongFieldFacetField() throws Exception { + doTestFieldFacetField("number_p_l", "number_p_l_dv", getSequentialStringArrayWithInts(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField("number_p_l_e", "number_p_l_e_dv", getSequentialStringArrayWithInts(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField("number_p_l", "number_p_l_dv", toStringArray(getRandomLongs(10, false))); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_l_e", "number_p_l_e_dv", toStringArray(getRandomLongs(10, false))); + } + + @Test + public void testLongFieldRangeFacet() { + String nonDocValuesField = "number_p_l" + (random().nextBoolean() ? "_e" : ""); + String docValuesField = nonDocValuesField + "_dv"; + int numValues = 10 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List sortedValues; + long max; + do { + values = getRandomLongs(numValues, false); + sortedValues = values.stream().sorted().collect(Collectors.toList()); + } while ((max = sortedValues.get(sortedValues.size() - 1)) + >= Long.MAX_VALUE - numValues); // leave room for rounding + long min = sortedValues.get(0); + BigInteger bigIntGap = + BigInteger.valueOf(max + numValues) + .subtract(BigInteger.valueOf(min)) + .divide(BigInteger.valueOf(numBuckets)); + long gap = bigIntGap.longValueExact(); + int[] bucketCount = new int[numBuckets]; + int bucketNum = 0; + long minBucketVal = min; + // System.err.println("min:" + min + " max: " + max + " gap: " + gap); + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + for (Long value : sortedValues) { + // System.err.println("value: " + value); + while (BigInteger.valueOf(value) + .subtract(BigInteger.valueOf(minBucketVal)) + .compareTo(bigIntGap) + > 0) { + ++bucketNum; + minBucketVal += gap; + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + minBucketVal); + } + ++bucketCount[bucketNum]; + } + + for (int i = 0; i < numValues; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + docValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i)))); + } + assertU(commit()); + + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + numValues + "']"; + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap)), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + bucketCount[i] + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv"), + testStrings); + } + + @Test + public void testLongFieldStats() { + int numValues = 10 * RANDOM_MULTIPLIER; + // don't produce numbers with exponents, since XPath comparison operators can't handle them + List values = getRandomLongs(numValues, false, 9999999L); + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + double min = (double) sortedValues.get(0); + double max = (double) sortedValues.get(sortedValues.size() - 1); + + String[] valArray = toStringArray(values); + doTestFieldStats("number_p_l", "number_p_l_dv", valArray, min, max, numValues, 1, 0D); + doTestFieldStats("number_p_l_e", "number_p_l_e_dv", valArray, min, max, numValues, 1, 0D); + doTestFieldStats("number_p_l_mv", "number_p_l_mv_dv", valArray, min, max, numValues, 1, 0D); + doTestFieldStats("number_p_l_e_mv", "number_p_l_e_mv_dv", valArray, min, max, numValues, 1, 0D); + } + + @Test + public void testLongFieldMultiValuedExactQuery() throws Exception { + String[] ints = toStringArray(getRandomInts(20, false)); + doTestFieldMultiValuedExactQuery("number_p_l_mv", ints); + doTestFieldMultiValuedExactQuery("number_p_l_e_mv", ints); + doTestFieldMultiValuedExactQuery("number_p_l_mv_dv", ints); + doTestFieldMultiValuedExactQuery("number_p_l_e_mv_dv", ints); + doTestFieldMultiValuedExactQuery("number_p_l_ni_mv_dv", ints); + } + + @Test + public void testLongFieldMultiValuedNonSearchableExactQuery() throws Exception { + String[] longs = toStringArray(getRandomLongs(20, false)); + doTestFieldMultiValuedExactQuery("number_p_l_ni_mv", longs, false); + doTestFieldMultiValuedExactQuery("number_p_l_ni_ns_mv", longs, false); + } + + @Test + public void testLongFieldMultiValuedReturn() throws Exception { + String[] longs = toStringArray(getRandomLongs(20, false)); + doTestFieldMultiValuedReturn("number_p_l_mv", "long", longs); + doTestFieldMultiValuedReturn("number_p_l_e_mv", "long", longs); + doTestFieldMultiValuedReturn("number_p_l_mv_dv", "long", longs); + doTestFieldMultiValuedReturn("number_p_l_e_mv_dv", "long", longs); + doTestFieldMultiValuedReturn("number_p_l_ni_mv_dv", "long", longs); + doTestFieldMultiValuedReturn("number_p_l_dv_ns_mv", "long", longs); + } + + @Test + public void testLongFieldMultiValuedRangeQuery() throws Exception { + String[] longs = + toStringArray(getRandomLongs(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedRangeQuery("number_p_l_mv", "long", longs); + doTestFieldMultiValuedRangeQuery("number_p_l_e_mv", "long", longs); + doTestFieldMultiValuedRangeQuery("number_p_l_ni_mv_dv", "long", longs); + doTestFieldMultiValuedRangeQuery("number_p_l_mv_dv", "long", longs); + doTestFieldMultiValuedRangeQuery("number_p_l_e_mv_dv", "long", longs); + } + + @Test + public void testLongFieldMultiValuedFacetField() throws Exception { + doTestFieldMultiValuedFacetField( + "number_p_l_mv", "number_p_l_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFacetField( + "number_p_l_e_mv", "number_p_l_e_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFacetField( + "number_p_l_mv", "number_p_l_mv_dv", toStringArray(getRandomLongs(20, false))); + doTestFieldMultiValuedFacetField( + "number_p_l_e_mv", "number_p_l_e_mv_dv", toStringArray(getRandomLongs(20, false))); + } + + @Test + public void testLongFieldMultiValuedRangeFacet() { + String nonDocValuesField = "number_p_l" + (random().nextBoolean() ? "_e" : "") + "_mv"; + String docValuesField = nonDocValuesField + "number_p_l_mv_dv"; + int numValues = 20 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List> sortedValues; + long max; + do { + values = getRandomLongs(numValues, false); + sortedValues = toAscendingPosVals(values, true); + } while ((max = sortedValues.get(sortedValues.size() - 1).val) + >= Long.MAX_VALUE - numValues); // leave room for rounding + long min = sortedValues.get(0).val; + long gap = + BigInteger.valueOf(max + numValues) + .subtract(BigInteger.valueOf(min)) + .divide(BigInteger.valueOf(numBuckets)) + .longValueExact(); + List> docIdBucket = new ArrayList<>(numBuckets); + for (int i = 0; i < numBuckets; ++i) { + docIdBucket.add(new HashSet<>()); + } + int bucketNum = 0; + long minBucketVal = min; + for (PosVal value : sortedValues) { + while (value.val - minBucketVal >= gap) { + ++bucketNum; + minBucketVal += gap; + } + docIdBucket.get(bucketNum).add(value.pos / 2); // each doc gets two consecutive values + } + for (int i = 0; i < numValues; i += 2) { + assertU( + adoc( + "id", + String.valueOf(i / 2), + docValuesField, + String.valueOf(values.get(i)), + docValuesField, + String.valueOf(values.get(i + 1)), + nonDocValuesField, + String.valueOf(values.get(i)), + nonDocValuesField, + String.valueOf(values.get(i + 1)))); + } + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + (numValues / 2) + "']"; + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "indent", + "on"), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal += gap, ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + minBucketVal + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "filter", + "indent", + "on"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + String.valueOf(min), + "facet.range.end", + String.valueOf(max), + "facet.range.gap", + String.valueOf(gap), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + } + + @Test + public void testLongFieldMultiValuedFunctionQuery() throws Exception { + doTestFieldMultiValuedFunctionQuery( + "number_p_l_mv", "number_p_l_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_l_e_mv", "number_p_l_e_mv_dv", getSequentialStringArrayWithInts(20)); + doTestFieldMultiValuedFunctionQuery( + "number_p_l_mv", + "number_p_l_mv_dv", + toStringArray(getRandomLongs(20, false).stream().sorted().collect(Collectors.toList()))); + doTestFieldMultiValuedFunctionQuery( + "number_p_l_e_mv", + "number_p_l_e_mv_dv", + toStringArray(getRandomLongs(20, false).stream().sorted().collect(Collectors.toList()))); + } + + @Test + public void testLongFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + doTestLongFieldsAtomicUpdates("number_p_l"); + doTestLongFieldsAtomicUpdates("number_p_l_e"); + doTestLongFieldsAtomicUpdates("number_p_l_dv"); + doTestLongFieldsAtomicUpdates("number_p_l_e_dv"); + doTestLongFieldsAtomicUpdates("number_p_l_dv_ns"); + } + + @Test + public void testMultiValuedLongFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + String[] longs = toStringArray(getRandomLongs(3, false)); + doTestMultiValuedFieldsAtomicUpdates("number_p_l_mv", "long", longs); + doTestMultiValuedFieldsAtomicUpdates("number_p_l_e_mv", "long", longs); + doTestMultiValuedFieldsAtomicUpdates("number_p_l_mv_dv", "long", longs); + doTestMultiValuedFieldsAtomicUpdates("number_p_l_e_mv_dv", "long", longs); + doTestMultiValuedFieldsAtomicUpdates("number_p_l_ni_mv_dv", "long", longs); + doTestMultiValuedFieldsAtomicUpdates("number_p_l_dv_ns_mv", "long", longs); + } + + @Test + public void testLongFieldSetQuery() { + doTestSetQueries("number_p_l", toStringArray(getRandomLongs(20, false)), false); + doTestSetQueries("number_p_l_dv", toStringArray(getRandomLongs(20, false)), false); + doTestSetQueries("number_p_l_mv", toStringArray(getRandomLongs(20, false)), true); + doTestSetQueries("number_p_l_mv_dv", toStringArray(getRandomLongs(20, false)), true); + doTestSetQueries("number_p_l_ni_dv", toStringArray(getRandomLongs(20, false)), false); + doTestSetQueries("number_p_l_e", toStringArray(getRandomLongs(20, false)), false); + doTestSetQueries("number_p_l_e_dv", toStringArray(getRandomLongs(20, false)), false); + doTestSetQueries("number_p_l_e_mv", toStringArray(getRandomLongs(20, false)), true); + doTestSetQueries("number_p_l_e_mv_dv", toStringArray(getRandomLongs(20, false)), true); + } + + @Test + public void testLongFieldNotIndexed() throws Exception { + String[] longs = toStringArray(getRandomLongs(10, false)); + doTestFieldNotIndexed("number_p_l_ni", longs); + doTestFieldNotIndexed("number_p_l_ni_mv", longs); + } + + // Date + + private String getRandomDateMaybeWithMath() { + long millis1 = random().nextLong() % MAX_DATE_EPOCH_MILLIS; + String date = Instant.ofEpochMilli(millis1).toString(); + if (random().nextBoolean()) { + long millis2 = random().nextLong() % MAX_DATE_EPOCH_MILLIS; + DateGapCeiling gap = new DateGapCeiling(millis2 - millis1); + date += gap.toString(); + } + return date; + } + + @Test + public void testDateFieldExactQuery() throws Exception { + String baseDate = getRandomDateMaybeWithMath(); + for (String field : + Arrays.asList( + "number_p_dt", + "number_p_dt_mv", + "number_p_dt_dv", + "number_p_dt_mv_dv", + "number_p_dt_ni_dv", + "number_p_dt_ni_ns_dv", + "number_p_dt_ni_mv_dv", + "number_p_dt_e", + "number_p_dt_e_mv", + "number_p_dt_e_dv", + "number_p_dt_e_mv_dv")) { + doTestDateFieldExactQuery(field, baseDate); + } + } + + @Test + public void testDateFieldNonSearchableExactQuery() throws Exception { + doTestDateFieldExactQuery("number_p_dt_ni", "1995-12-31T23:59:59Z", false); + doTestDateFieldExactQuery("number_p_dt_ni_ns", "1995-12-31T23:59:59Z", false); + } + + @Test + public void testDateFieldReturn() throws Exception { + int numValues = 10 * RANDOM_MULTIPLIER; + String[] dates = toStringArray(getRandomInstants(numValues, false)); + doTestFieldReturn("number_p_dt", "date", dates); + doTestFieldReturn("number_p_dt_e", "date", dates); + doTestFieldReturn("number_p_dt_dv", "date", dates); + doTestFieldReturn("number_p_dt_e_dv", "date", dates); + doTestFieldReturn("number_p_dt_dv_ns", "date", dates); + } + + @Test + public void testDateFieldRangeQuery() throws Exception { + doTestDateFieldRangeQuery("number_p_dt"); + doTestDateFieldRangeQuery("number_p_dt_e"); + doTestDateFieldRangeQuery("number_p_dt_dv"); + doTestDateFieldRangeQuery("number_p_dt_e_dv"); + doTestDateFieldRangeQuery("number_p_dt_ni_ns_dv"); + } + + @Test + public void testDateFieldNonSearchableRangeQuery() throws Exception { + doTestFieldNonSearchableRangeQuery( + "number_p_dt_ni", toStringArray(getRandomInstants(1, false))); + doTestFieldNonSearchableRangeQuery( + "number_p_dt_ni_ns", toStringArray(getRandomInstants(1, false))); + int numValues = 2 * RANDOM_MULTIPLIER; + doTestFieldNonSearchableRangeQuery( + "number_p_dt_ni_ns_mv", toStringArray(getRandomInstants(numValues, false))); + } + + @Test + public void testDateFieldSortAndFunction() throws Exception { + final SortedSet regexToTest = dynFieldRegexesForType(DateField.class); + final List sequential = Arrays.asList(getSequentialStringArrayWithDates(10)); + final List randomDates = getRandomInstants(10, false); + final List randomDatesMissing = getRandomInstants(10, true); + + for (String r : + Arrays.asList( + "*_p_dt", + "*_p_dt_e", + "*_p_dt_dv", + "*_p_dt_e_dv", + "*_p_dt_dv_ns", + "*_p_dt_ni_dv", + "*_p_dt_ni_dv_ns", + "*_p_dt_ni_ns_dv")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomDates); + doTestDateFieldFunctionQuery(field); + } + for (String r : + Arrays.asList( + "*_p_dt_smf", + "*_p_dt_e_smf", + "*_p_dt_dv_smf", + "*_p_dt_e_dv_smf", + "*_p_dt_ni_dv_smf", + "*_p_dt_sml", + "*_p_dt_e_sml", + "*_p_dt_dv_sml", + "*_p_dt_e_dv_sml", + "*_p_dt_ni_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomDatesMissing); + doTestDateFieldFunctionQuery(field); + } + + for (String r : Arrays.asList("*_p_dt_ni", "*_p_dt_ni_ns")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", "1995-12-31T23:59:59Z"); + doTestFieldFunctionQueryError(field, "w/o docValues", "1995-12-31T23:59:59Z"); + } + + // multivalued, no docvalues + for (String r : + Arrays.asList( + "*_p_dt_mv", + "*_p_dt_e_mv", + "*_p_dt_ni_mv", + "*_p_dt_ni_ns_mv", + "*_p_dt_mv_smf", + "*_p_dt_e_mv_smf", + "*_p_dt_mv_sml", + "*_p_dt_e_mv_sml")) { + + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + doTestFieldSortError(field, "w/o docValues", "1995-12-31T23:59:59Z"); + doTestFieldSortError(field, "w/o docValues", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); + doTestFieldFunctionQueryError(field, "multivalued", "1995-12-31T23:59:59Z"); + doTestFieldFunctionQueryError( + field, "multivalued", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); + } + + // multivalued, w/ docValues + for (String r : + Arrays.asList( + "*_p_dt_ni_mv_dv", + "*_p_dt_ni_dv_ns_mv", + "*_p_dt_dv_ns_mv", + "*_p_dt_mv_dv", + "*_p_dt_e_mv_dv", + "*_p_dt_e_mv_dv_smf", + "*_p_dt_e_mv_dv_sml", + "*_p_dt_mv_dv_smf", + "*_p_dt_ni_mv_dv_smf", + "*_p_dt_mv_dv_sml", + "*_p_dt_ni_mv_dv_sml")) { + assertTrue(r, regexToTest.remove(r)); + String field = r.replace("*", "number"); + + // NOTE: only testing one value per doc here, but TestMinMaxOnMultiValuedField + // covers this in more depth + doTestFieldSort(field, sequential); + doTestFieldSort(field, randomDates); + + // value source (w/o field(...,min|max)) usage should still error... + doTestFieldFunctionQueryError(field, "multivalued", "1995-12-31T23:59:59Z"); + doTestFieldFunctionQueryError( + field, "multivalued", "1995-12-31T23:59:59Z", "2000-12-31T23:59:59Z"); + } + assertEquals("Missing types in the test", Collections.emptySet(), regexToTest); + } + + @Test + public void testDateFieldFacetField() throws Exception { + doTestFieldFacetField("number_p_dt", "number_p_dt_dv", getSequentialStringArrayWithDates(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_dt_e", "number_p_dt_e_dv", getSequentialStringArrayWithDates(10)); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_dt", "number_p_dt_dv", toStringArray(getRandomInstants(10, false))); + clearIndex(); + assertU(commit()); + doTestFieldFacetField( + "number_p_dt_e", "number_p_dt_e_dv", toStringArray(getRandomInstants(10, false))); + } + + private static class DateGapCeiling { + String calendarUnit = "MILLIS"; + long inCalendarUnits; + boolean negative = false; + + /** Maximize calendar unit size given initialGapMillis; performs ceiling on each conversion */ + DateGapCeiling(long initialGapMillis) { + negative = initialGapMillis < 0; + inCalendarUnits = Math.abs(initialGapMillis); + if (inCalendarUnits >= 1000L) { + calendarUnit = "SECS"; + inCalendarUnits = (inCalendarUnits + 999L) / 1000L; + if (inCalendarUnits >= 60L) { + calendarUnit = "MINUTES"; + inCalendarUnits = (inCalendarUnits + 59L) / 60L; + if (inCalendarUnits >= 60L) { + calendarUnit = "HOURS"; + inCalendarUnits = (inCalendarUnits + 59L) / 60L; + if (inCalendarUnits >= 24L) { + calendarUnit = "DAYS"; + inCalendarUnits = (inCalendarUnits + 23L) / 24L; + if (inCalendarUnits >= 12L) { + calendarUnit = "MONTHS"; + inCalendarUnits = (inCalendarUnits + 11L) / 12L; + // 487 = 365.25 / 12 * 16 (365.25 days/year, -ish) + if ((inCalendarUnits * 16) >= 487) { + calendarUnit = "YEARS"; + inCalendarUnits = (16L * inCalendarUnits + 486) / 487L; + } + } + } + } + } + } + } + + @Override + public String toString() { + return (negative ? "-" : "+") + inCalendarUnits + calendarUnit; + } + + public long addTo(long millis) { + // Instant.plus() doesn't work with estimated durations (MONTHS and YEARS) + LocalDateTime time = + LocalDateTime.ofInstant(Instant.ofEpochMilli(millis), ZoneOffset.ofHours(0)); + if (negative) { + time = time.minus(inCalendarUnits, DateMathParser.CALENDAR_UNITS.get(calendarUnit)); + } else { + time = time.plus(inCalendarUnits, DateMathParser.CALENDAR_UNITS.get(calendarUnit)); + } + return time.atZone(ZoneOffset.ofHours(0)).toInstant().toEpochMilli(); + } + } + + @Test + public void testDateFieldRangeFacet() { + String nonDocValuesField = "number_p_dt" + (random().nextBoolean() ? "_e" : ""); + String docValuesField = nonDocValuesField + "_dv"; + int numValues = 10 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values, sortedValues; + long min, max; + DateGapCeiling gap; + do { + values = getRandomLongs(numValues, false, MAX_DATE_EPOCH_MILLIS); + sortedValues = values.stream().sorted().collect(Collectors.toList()); + min = sortedValues.get(0); + max = sortedValues.get(sortedValues.size() - 1); + } while (max > MAX_DATE_EPOCH_MILLIS || min < MIN_DATE_EPOCH_MILLIS); + long initialGap = + BigInteger.valueOf(max) + .subtract(BigInteger.valueOf(min)) + .divide(BigInteger.valueOf(numBuckets)) + .longValueExact(); + gap = + new DateGapCeiling( + BigInteger.valueOf(max + initialGap) + .subtract(BigInteger.valueOf(min)) // padding for rounding + .divide(BigInteger.valueOf(numBuckets)) + .longValueExact()); + int[] bucketCount = new int[numBuckets]; + int bucketNum = 0; + long minBucketVal = min; + // System.err.println("min:" + Instant.ofEpochMilli(min) + " max: " + + // Instant.ofEpochMilli(max) + " gap: " + gap); + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + + // Instant.ofEpochMilli(minBucketVal)); + for (long value : sortedValues) { + // System.err.println("value: " + Instant.ofEpochMilli(value)); + while (value >= gap.addTo(minBucketVal)) { + ++bucketNum; + minBucketVal = gap.addTo(minBucketVal); + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + + // Instant.ofEpochMilli(minBucketVal)); + } + ++bucketCount[bucketNum]; + } + + for (int i = 0; i < numValues; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + docValuesField, + Instant.ofEpochMilli(values.get(i)).toString(), + nonDocValuesField, + Instant.ofEpochMilli(values.get(i)).toString())); + } + assertU(commit()); + + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + numValues + "']"; + minBucketVal = min; + for (int i = 0; i < numBuckets; ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + Instant.ofEpochMilli(minBucketVal) + + "'][.='" + + bucketCount[i] + + "']"; + minBucketVal = gap.addTo(minBucketVal); + } + long maxPlusGap = gap.addTo(max); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + Instant.ofEpochMilli(min).toString(), + "facet.range.end", + Instant.ofEpochMilli(maxPlusGap).toString(), + "facet.range.gap", + gap.toString()), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + Instant.ofEpochMilli(min).toString(), + "facet.range.end", + Instant.ofEpochMilli(maxPlusGap).toString(), + "facet.range.gap", + gap.toString(), + "facet.range.method", + "dv"), + testStrings); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + minBucketVal = min; + for (int i = 0; i < numBuckets; ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + Instant.ofEpochMilli(minBucketVal).toString() + + "'][.='" + + bucketCount[i] + + "']"; + minBucketVal = gap.addTo(minBucketVal); + } + maxPlusGap = gap.addTo(max); + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + Instant.ofEpochMilli(min).toString(), + "facet.range.end", + Instant.ofEpochMilli(maxPlusGap).toString(), + "facet.range.gap", + gap.toString(), + "facet.range.method", + "filter"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + Instant.ofEpochMilli(min).toString(), + "facet.range.end", + Instant.ofEpochMilli(maxPlusGap).toString(), + "facet.range.gap", + gap.toString(), + "facet.range.method", + "dv"), + testStrings); + } + + @Test + public void testDateFieldStats() { + String[] randomSortedDates = toAscendingStringArray(getRandomInstants(10, false), true); + doTestDateFieldStats("number_p_dt", "number_p_dt_dv", randomSortedDates); + doTestDateFieldStats("number_p_dt_e", "number_p_dt_e_dv", randomSortedDates); + doTestDateFieldStats("number_p_dt_mv", "number_p_dt_mv_dv", randomSortedDates); + doTestDateFieldStats("number_p_dt_e_mv", "number_p_dt_e_mv_dv", randomSortedDates); + } + + @Test + public void testDateFieldMultiValuedExactQuery() throws Exception { + String[] dates = toStringArray(getRandomInstants(20, false)); + doTestFieldMultiValuedExactQuery("number_p_dt_mv", dates); + doTestFieldMultiValuedExactQuery("number_p_dt_e_mv", dates); + doTestFieldMultiValuedExactQuery("number_p_dt_mv_dv", dates); + doTestFieldMultiValuedExactQuery("number_p_dt_e_mv_dv", dates); + doTestFieldMultiValuedExactQuery("number_p_dt_ni_mv_dv", dates); + } + + @Test + public void testDateFieldMultiValuedNonSearchableExactQuery() throws Exception { + String[] dates = toStringArray(getRandomInstants(20, false)); + doTestFieldMultiValuedExactQuery("number_p_dt_ni_mv", dates, false); + doTestFieldMultiValuedExactQuery("number_p_dt_ni_ns_mv", dates, false); + } + + @Test + public void testDateFieldMultiValuedReturn() throws Exception { + String[] dates = toStringArray(getRandomInstants(20, false)); + doTestFieldMultiValuedReturn("number_p_dt_mv", "date", dates); + doTestFieldMultiValuedReturn("number_p_dt_e_mv", "date", dates); + doTestFieldMultiValuedReturn("number_p_dt_mv_dv", "date", dates); + doTestFieldMultiValuedReturn("number_p_dt_e_mv_dv", "date", dates); + doTestFieldMultiValuedReturn("number_p_dt_ni_mv_dv", "date", dates); + doTestFieldMultiValuedReturn("number_p_dt_dv_ns_mv", "date", dates); + } + + @Test + public void testDateFieldMultiValuedRangeQuery() throws Exception { + String[] dates = + toStringArray(getRandomInstants(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedRangeQuery("number_p_dt_mv", "date", dates); + doTestFieldMultiValuedRangeQuery("number_p_dt_e_mv", "date", dates); + doTestFieldMultiValuedRangeQuery("number_p_dt_mv_dv", "date", dates); + doTestFieldMultiValuedRangeQuery("number_p_dt_e_mv_dv", "date", dates); + doTestFieldMultiValuedRangeQuery("number_p_dt_ni_mv_dv", "date", dates); + } + + @Test + public void testDateFieldMultiValuedFacetField() throws Exception { + doTestFieldMultiValuedFacetField( + "number_p_dt_mv", "number_p_dt_mv_dv", getSequentialStringArrayWithDates(20)); + doTestFieldMultiValuedFacetField( + "number_p_dt_e_mv", "number_p_dt_e_mv_dv", getSequentialStringArrayWithDates(20)); + doTestFieldMultiValuedFacetField( + "number_p_dt_mv", "number_p_dt_mv_dv", toStringArray(getRandomInstants(20, false))); + doTestFieldMultiValuedFacetField( + "number_p_dt_e_mv", "number_p_dt_e_mv_dv", toStringArray(getRandomInstants(20, false))); + } + + @Test + public void testDateFieldMultiValuedRangeFacet() { + String nonDocValuesField = "number_p_dt" + (random().nextBoolean() ? "_e" : "") + "_mv"; + String docValuesField = nonDocValuesField + "_dv"; + SchemaField dvSchemaField = h.getCore().getLatestSchema().getField(docValuesField); + assertTrue(dvSchemaField.multiValued()); + assertTrue(dvSchemaField.hasDocValues()); + assertTrue(dvSchemaField.getType() instanceof NumericField); + + SchemaField nonDvSchemaField = h.getCore().getLatestSchema().getField(nonDocValuesField); + assertTrue(nonDvSchemaField.multiValued()); + assertFalse(nonDvSchemaField.hasDocValues()); + assertTrue(nonDvSchemaField.getType() instanceof NumericField); + + int numValues = 20 * RANDOM_MULTIPLIER; + int numBuckets = numValues / 2; + List values; + List> sortedValues; + long min, max; + do { + values = getRandomLongs(numValues, false, MAX_DATE_EPOCH_MILLIS); + sortedValues = toAscendingPosVals(values, true); + min = sortedValues.get(0).val; + max = sortedValues.get(sortedValues.size() - 1).val; + } while (max > MAX_DATE_EPOCH_MILLIS || min < MIN_DATE_EPOCH_MILLIS); + long initialGap = + BigInteger.valueOf(max) + .subtract(BigInteger.valueOf(min)) + .divide(BigInteger.valueOf(numBuckets)) + .longValueExact(); + DateGapCeiling gap = + new DateGapCeiling( + BigInteger.valueOf(max + initialGap) + .subtract(BigInteger.valueOf(min)) // padding for rounding + .divide(BigInteger.valueOf(numBuckets)) + .longValueExact()); + List> docIdBucket = new ArrayList<>(numBuckets); + for (int i = 0; i < numBuckets; ++i) { + docIdBucket.add(new HashSet<>()); + } + int bucketNum = 0; + long minBucketVal = min; + // System.err.println("min:" + Instant.ofEpochMilli(min) + " max: " + + // Instant.ofEpochMilli(max) + " gap: " + gap); + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + + // Instant.ofEpochMilli(minBucketVal)); + for (PosVal value : sortedValues) { + // System.err.println("value: " + Instant.ofEpochMilli(value.val)); + while (value.val >= gap.addTo(minBucketVal)) { + ++bucketNum; + minBucketVal = gap.addTo(minBucketVal); + // System.err.println("bucketNum: " + bucketNum + " minBucketVal: " + + // Instant.ofEpochMilli(minBucketVal)); + } + docIdBucket.get(bucketNum).add(value.pos / 2); // each doc gets two consecutive values + } + for (int i = 0; i < numValues; i += 2) { + assertU( + adoc( + "id", + String.valueOf(i / 2), + docValuesField, + Instant.ofEpochMilli(values.get(i)).toString(), + docValuesField, + Instant.ofEpochMilli(values.get(i + 1)).toString(), + nonDocValuesField, + Instant.ofEpochMilli(values.get(i)).toString(), + nonDocValuesField, + Instant.ofEpochMilli(values.get(i + 1)).toString())); + } + assertU(commit()); + + String minDate = Instant.ofEpochMilli(min).toString(); + String maxDate = Instant.ofEpochMilli(max).toString(); + String[] testStrings = new String[numBuckets + 1]; + testStrings[numBuckets] = "//*[@numFound='" + (numValues / 2) + "']"; + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal = gap.addTo(minBucketVal), ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + docValuesField + + "']/lst[@name='counts']/int[@name='" + + Instant.ofEpochMilli(minBucketVal) + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + minDate, + "facet.range.end", + maxDate, + "facet.range.gap", + gap.toString(), + "indent", + "on"), + testStrings); + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + docValuesField, + "facet.range.start", + minDate, + "facet.range.end", + maxDate, + "facet.range.gap", + gap.toString(), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + + minBucketVal = min; + for (int i = 0; i < numBuckets; minBucketVal = gap.addTo(minBucketVal), ++i) { + testStrings[i] = + "//lst[@name='facet_counts']/lst[@name='facet_ranges']/lst[@name='" + + nonDocValuesField + + "']/lst[@name='counts']/int[@name='" + + Instant.ofEpochMilli(minBucketVal) + + "'][.='" + + docIdBucket.get(i).size() + + "']"; + } + // Range Faceting with method = filter should work + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + minDate, + "facet.range.end", + maxDate, + "facet.range.gap", + gap.toString(), + "facet.range.method", + "filter", + "indent", + "on"), + testStrings); + // this should actually use filter method instead of dv + assertQ( + req( + "q", + "*:*", + "facet", + "true", + "facet.range", + nonDocValuesField, + "facet.range.start", + minDate, + "facet.range.end", + maxDate, + "facet.range.gap", + gap.toString(), + "facet.range.method", + "dv", + "indent", + "on"), + testStrings); + } + + @Test + public void testDateFieldMultiValuedFunctionQuery() throws Exception { + String[] dates = + toStringArray(getRandomInstants(20, false).stream().sorted().collect(Collectors.toList())); + doTestFieldMultiValuedFunctionQuery("number_p_dt_mv", "number_p_dt_mv_dv", dates); + doTestFieldMultiValuedFunctionQuery("number_p_dt_e_mv", "number_p_dt_e_mv_dv", dates); + } + + @Test + public void testDateFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + doTestDateFieldsAtomicUpdates("number_p_dt"); + doTestDateFieldsAtomicUpdates("number_p_dt_e"); + doTestDateFieldsAtomicUpdates("number_p_dt_dv"); + doTestDateFieldsAtomicUpdates("number_p_dt_e_dv"); + doTestDateFieldsAtomicUpdates("number_p_dt_dv_ns"); + } + + @Test + public void testMultiValuedDateFieldsAtomicUpdates() throws Exception { + if (!Boolean.getBoolean("solr.index.updatelog.enabled")) { + return; + } + String[] dates = + getRandomLongs(3, false, MAX_DATE_EPOCH_MILLIS).stream() + .map(Instant::ofEpochMilli) + .map(Object::toString) + .toArray(String[]::new); + doTestMultiValuedFieldsAtomicUpdates("number_p_dt_mv", "date", dates); + doTestMultiValuedFieldsAtomicUpdates("number_p_dt_e_mv", "date", dates); + doTestMultiValuedFieldsAtomicUpdates("number_p_dt_mv_dv", "date", dates); + doTestMultiValuedFieldsAtomicUpdates("number_p_dt_e_mv_dv", "date", dates); + doTestMultiValuedFieldsAtomicUpdates("number_p_dt_ni_mv_dv", "date", dates); + doTestMultiValuedFieldsAtomicUpdates("number_p_dt_dv_ns_mv", "date", dates); + } + + @Test + public void testDateFieldSetQuery() { + doTestSetQueries("number_p_dt", toStringArray(getRandomInstants(20, false)), false); + doTestSetQueries("number_p_dt_dv", toStringArray(getRandomInstants(20, false)), false); + doTestSetQueries("number_p_dt_mv", toStringArray(getRandomInstants(20, false)), true); + doTestSetQueries("number_p_dt_mv_dv", toStringArray(getRandomInstants(20, false)), true); + doTestSetQueries("number_p_dt_ni_dv", toStringArray(getRandomInstants(20, false)), false); + doTestSetQueries("number_p_dt_e", toStringArray(getRandomInstants(20, false)), false); + doTestSetQueries("number_p_dt_e_dv", toStringArray(getRandomInstants(20, false)), false); + doTestSetQueries("number_p_dt_e_mv", toStringArray(getRandomInstants(20, false)), true); + doTestSetQueries("number_p_dt_e_mv_dv", toStringArray(getRandomInstants(20, false)), true); + } + + @Test + public void testDateFieldNotIndexed() throws Exception { + String[] dates = toStringArray(getRandomInstants(10, false)); + doTestFieldNotIndexed("number_p_dt_ni", dates); + doTestFieldNotIndexed("number_p_dt_ni_mv", dates); + } + + @Test + public void testIndexOrDocValuesQuery() { + String[] fieldTypeNames = new String[] {"_p_i", "_p_l", "_p_d", "_p_f", "_p_dt"}; + FieldType[] fieldTypes = + new FieldType[] { + new IntField(), new LongField(), new DoubleField(), new FloatField(), new DateField() + }; + String[] ints = + toStringArray(getRandomInts(2, false).stream().sorted().collect(Collectors.toList())); + String[] longs = + toStringArray(getRandomLongs(2, false).stream().sorted().collect(Collectors.toList())); + String[] doubles = + toStringArray(getRandomDoubles(2, false).stream().sorted().collect(Collectors.toList())); + String[] floats = + toStringArray(getRandomFloats(2, false).stream().sorted().collect(Collectors.toList())); + String[] dates = + toStringArray(getRandomInstants(2, false).stream().sorted().collect(Collectors.toList())); + String[] min = new String[] {ints[0], longs[0], doubles[0], floats[0], dates[0]}; + String[] max = new String[] {ints[1], longs[1], doubles[1], floats[1], dates[1]}; + assertTrue( + fieldTypeNames.length == fieldTypes.length + && fieldTypeNames.length == max.length + && fieldTypeNames.length == min.length); + Query q; + for (int i = 0; i < fieldTypeNames.length; i++) { + SchemaField fieldIndexed = h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i]); + SchemaField fieldIndexedAndDv = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_dv"); + SchemaField fieldIndexedMv = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_mv"); + SchemaField fieldIndexedAndDvMv = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_mv_dv"); + assertTrue( + fieldTypes[i].getRangeQuery(null, fieldIndexed, min[i], max[i], true, true) + instanceof PointRangeQuery); + q = fieldTypes[i].getRangeQuery(null, fieldIndexedAndDv, min[i], max[i], true, true); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointRangeQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getRangeQuery(null, fieldIndexedMv, min[i], max[i], true, true) + instanceof PointRangeQuery); + q = fieldTypes[i].getRangeQuery(null, fieldIndexedAndDvMv, min[i], max[i], true, true); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointRangeQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getFieldQuery(null, fieldIndexed, min[i]) instanceof PointRangeQuery); + q = fieldTypes[i].getFieldQuery(null, fieldIndexedAndDv, min[i]); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointRangeQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getFieldQuery(null, fieldIndexedMv, min[i]) instanceof PointRangeQuery); + q = fieldTypes[i].getFieldQuery(null, fieldIndexedAndDvMv, min[i]); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointRangeQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getSetQuery(null, fieldIndexed, List.of(min[i], max[i])) + instanceof PointInSetQuery); + q = fieldTypes[i].getSetQuery(null, fieldIndexedAndDv, List.of(min[i], max[i])); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointInSetQuery); + assertEquals( + "SortedNumericDocValuesSetQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getSetQuery(null, fieldIndexedMv, List.of(min[i], max[i])) + instanceof PointInSetQuery); + q = fieldTypes[i].getSetQuery(null, fieldIndexedAndDvMv, List.of(min[i], max[i])); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointInSetQuery); + assertEquals( + "SortedNumericDocValuesSetQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + } + for (int i = 0; i < fieldTypeNames.length; i++) { + SchemaField fieldIndexed = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_e"); + SchemaField fieldIndexedAndDv = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_e_dv"); + SchemaField fieldIndexedMv = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_e_mv"); + SchemaField fieldIndexedAndDvMv = + h.getCore().getLatestSchema().getField("foo" + fieldTypeNames[i] + "_e_mv_dv"); + assertTrue( + fieldTypes[i].getRangeQuery(null, fieldIndexed, min[i], max[i], true, true) + instanceof PointRangeQuery); + q = fieldTypes[i].getRangeQuery(null, fieldIndexedAndDv, min[i], max[i], true, true); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointRangeQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getRangeQuery(null, fieldIndexedMv, min[i], max[i], true, true) + instanceof PointRangeQuery); + q = fieldTypes[i].getRangeQuery(null, fieldIndexedAndDvMv, min[i], max[i], true, true); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof PointRangeQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue(fieldTypes[i].getFieldQuery(null, fieldIndexed, min[i]) instanceof TermQuery); + q = fieldTypes[i].getFieldQuery(null, fieldIndexedAndDv, min[i]); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof TermQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue(fieldTypes[i].getFieldQuery(null, fieldIndexedMv, min[i]) instanceof TermQuery); + q = fieldTypes[i].getFieldQuery(null, fieldIndexedAndDvMv, min[i]); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof TermQuery); + assertEquals( + "SortedNumericDocValuesRangeQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getSetQuery(null, fieldIndexed, List.of(min[i], max[i])) + instanceof TermInSetQuery); + q = fieldTypes[i].getSetQuery(null, fieldIndexedAndDv, List.of(min[i], max[i])); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof TermInSetQuery); + assertEquals( + "SortedNumericDocValuesSetQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + assertTrue( + fieldTypes[i].getSetQuery(null, fieldIndexedMv, List.of(min[i], max[i])) + instanceof TermInSetQuery); + q = fieldTypes[i].getSetQuery(null, fieldIndexedAndDvMv, List.of(min[i], max[i])); + assertTrue(q instanceof IndexOrDocValuesQuery); + assertTrue(((IndexOrDocValuesQuery) q).getIndexQuery() instanceof TermInSetQuery); + assertEquals( + "SortedNumericDocValuesSetQuery", + ((IndexOrDocValuesQuery) q).getRandomAccessQuery().getClass().getSimpleName()); + } + } + + public void testInternals() throws IOException { + String[] types = new String[] {"i", "l", "f", "d", "dt"}; + String[][] values = + new String[][] { + toStringArray(getRandomInts(10, false)), + toStringArray(getRandomLongs(10, false)), + toStringArray(getRandomFloats(10, false)), + toStringArray(getRandomDoubles(10, false)), + toStringArray(getRandomInstants(10, false)) + }; + assertEquals(types.length, values.length); + Set typesTested = new HashSet<>(); + for (int i = 0; i < types.length; ++i) { + for (String suffix : FIELD_SUFFIXES) { + doTestInternals("number_p_" + types[i] + suffix, values[i]); + typesTested.add("*_p_" + types[i] + suffix); + } + } + assertEquals( + "Missing types in the test", dynFieldRegexesForType(NumericField.class), typesTested); + } + + // Helper methods + + /** + * Given a FieldType, return the list of DynamicField 'regexes' for all declared DynamicFields + * that use that FieldType. + * + * @see IndexSchema#getDynamicFields + * @see DynamicField#getRegex + */ + private static SortedSet dynFieldRegexesForType(final Class clazz) { + SortedSet typesToTest = new TreeSet<>(); + for (DynamicField dynField : h.getCore().getLatestSchema().getDynamicFields()) { + if (clazz.isInstance(dynField.getPrototype().getType())) { + typesToTest.add(dynField.getRegex()); + } + } + return typesToTest; + } + + private List getRandomList(int length, boolean missingVals, Supplier randomVal) { + List list = new ArrayList<>(length); + for (int i = 0; i < length; ++i) { + T val = null; + // Sometimes leave val as null when we're producing missing values + if (missingVals == false || usually()) { + val = randomVal.get(); + } + list.add(val); + } + return list; + } + + private List getRandomDoubles(int length, boolean missingVals) { + return getRandomList( + length, + missingVals, + () -> { + Double d = Double.NaN; + while (d.isNaN()) { + d = Double.longBitsToDouble(random().nextLong()); + } + return d; + }); + } + + private List getRandomFloats(int length, boolean missingVals) { + return getRandomList( + length, + missingVals, + () -> { + Float f = Float.NaN; + while (f.isNaN()) { + f = Float.intBitsToFloat(random().nextInt()); + } + return f; + }); + } + + private List getRandomInts(int length, boolean missingVals, int boundPosNeg) { + assertTrue(boundPosNeg > 0L); + return getRandomList( + length, + missingVals, + () -> (random().nextBoolean() ? 1 : -1) * random().nextInt(boundPosNeg)); + } + + private List getRandomInts(int length, boolean missingVals) { + return getRandomList(length, missingVals, () -> random().nextInt()); + } + + private List getRandomLongs(int length, boolean missingVals, long boundPosNeg) { + assertTrue(boundPosNeg > 0L); + return getRandomList( + length, + missingVals, + () -> random().nextLong() % boundPosNeg); // see Random.nextInt(int bound) + } + + private List getRandomLongs(int length, boolean missingVals) { + return getRandomList(length, missingVals, () -> random().nextLong()); + } + + private List getRandomInstants(int length, boolean missingVals) { + return getRandomList(length, missingVals, () -> Instant.ofEpochMilli(random().nextLong())); + } + + private String[] getSequentialStringArrayWithInts(int length) { + String[] arr = new String[length]; + for (int i = 0; i < length; i++) { + arr[i] = String.valueOf(i); + } + return arr; + } + + private String[] getSequentialStringArrayWithDates(int length) { + assertTrue(length < 60); + String[] arr = new String[length]; + for (int i = 0; i < length; i++) { + arr[i] = String.format(Locale.ROOT, "1995-12-11T19:59:%02dZ", i); + } + return arr; + } + + private String[] getSequentialStringArrayWithDoubles(int length) { + String[] arr = new String[length]; + for (int i = 0; i < length; i++) { + arr[i] = String.format(Locale.ROOT, "%d.0", i); + } + return arr; + } + + private void doTestFieldNotIndexed(String field, String[] values) throws IOException { + assertTrue(values.length == 10); + // test preconditions + SchemaField sf = h.getCore().getLatestSchema().getField(field); + assertFalse("Field should be indexed=false", sf.indexed()); + assertFalse("Field should be docValues=false", sf.hasDocValues()); + + for (int i = 0; i < 10; i++) { + assertU(adoc("id", String.valueOf(i), field, values[i])); + } + assertU(commit()); + assertQ(req("q", "*:*"), "//*[@numFound='10']"); + assertQ( + "Can't search on index=false docValues=false field", + req("q", field + ":[* TO *]"), + "//*[@numFound='0']"); + h.getCore() + .withSearcher( + searcher -> { + IndexReader ir = searcher.getIndexReader(); + assertEquals( + "Field " + field + " should have no point values", + 0, + PointValues.size(ir, field)); + return null; + }); + } + + private void doTestIntFieldExactQuery(final String field, final boolean testLong) + throws Exception { + doTestIntFieldExactQuery(field, testLong, true); + } + + private String getTestString(boolean searchable, int numFound) { + return "//*[@numFound='" + (searchable ? Integer.toString(numFound) : "0") + "']"; + } + + /** + * @param field the field to use for indexing and searching against + * @param testLong set to true if "field" is expected to support long values, false if only + * integers + * @param searchable set to true if searches against "field" should succeed, false if field is + * only stored and searches should always get numFound=0 + */ + private void doTestIntFieldExactQuery( + final String field, final boolean testLong, final boolean searchable) { + int numValues = 10 * RANDOM_MULTIPLIER; + Map randCount = new HashMap<>(numValues); + String[] rand = + testLong + ? toStringArray(getRandomLongs(numValues, false)) + : toStringArray(getRandomInts(numValues, false)); + for (int i = 0; i < numValues; i++) { + randCount.merge(rand[i], 1, (a, b) -> a + b); // count unique values + assertU(adoc("id", String.valueOf(i), field, rand[i])); + } + assertU(commit()); + + for (int i = 0; i < numValues; i++) { + assertQ( + req( + "q", + field + ":" + (rand[i].startsWith("-") ? "\\" : "") + rand[i], + "fl", + "id," + field), + getTestString(searchable, randCount.get(rand[i]))); + } + + StringBuilder builder = new StringBuilder(); + for (String value : randCount.keySet()) { + if (builder.length() != 0) { + builder.append(" OR "); + } + if (value.startsWith("-")) { + builder.append("\\"); // escape negative sign + } + builder.append(value); + } + assertQ( + req("debug", "true", "q", field + ":(" + builder + ")"), + getTestString(searchable, numValues)); + + assertU( + adoc("id", String.valueOf(Integer.MAX_VALUE), field, String.valueOf(Integer.MAX_VALUE))); + assertU(commit()); + assertQ( + req("q", field + ":" + Integer.MAX_VALUE, "fl", "id, " + field), + getTestString(searchable, 1)); + + clearIndex(); + assertU(commit()); + } + + private void doTestFieldReturn(String field, String type, String[] values) { + SchemaField sf = h.getCore().getLatestSchema().getField(field); + assertTrue( + "Unexpected field definition for " + field, + sf.stored() || (sf.hasDocValues() && sf.useDocValuesAsStored())); + for (int i = 0; i < values.length; i++) { + assertU(adoc("id", String.valueOf(i), field, values[i])); + } + // Check using RTG + if (Boolean.getBoolean("solr.index.updatelog.enabled")) { + for (int i = 0; i < values.length; i++) { + assertQ( + req("qt", "/get", "id", String.valueOf(i)), + "//doc/" + type + "[@name='" + field + "'][.='" + values[i] + "']"); + } + } + assertU(commit()); + String[] expected = new String[values.length + 1]; + expected[0] = "//*[@numFound='" + values.length + "']"; + for (int i = 0; i < values.length; i++) { + expected[i + 1] = + "//result/doc[str[@name='id']='" + + i + + "']/" + + type + + "[@name='" + + field + + "'][.='" + + values[i] + + "']"; + } + assertQ(req("q", "*:*", "fl", "id, " + field, "rows", String.valueOf(values.length)), expected); + + // Check using RTG + if (Boolean.getBoolean("solr.index.updatelog.enabled")) { + for (int i = 0; i < values.length; i++) { + assertQ( + req("qt", "/get", "id", String.valueOf(i)), + "//doc/" + type + "[@name='" + field + "'][.='" + values[i] + "']"); + } + } + clearIndex(); + assertU(commit()); + } + + private void doTestFieldNonSearchableRangeQuery(String fieldName, String... values) { + for (int i = 9; i >= 0; i--) { + SolrInputDocument doc = sdoc("id", String.valueOf(i)); + for (String value : values) { + doc.addField(fieldName, value); + } + assertU(adoc(doc)); + } + assertU(commit()); + assertQ( + req("q", fieldName + ":[* TO *]", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='0']"); + } + + private void doTestIntFieldRangeQuery(String fieldName, String type, boolean testLong) { + for (int i = 9; i >= 0; i--) { + assertU(adoc("id", String.valueOf(i), fieldName, String.valueOf(i))); + } + assertU(commit()); + assertQ( + req("q", fieldName + ":[0 TO 3]", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='2']", + "//result/doc[4]/" + type + "[@name='" + fieldName + "'][.='3']"); + + assertQ( + req("q", fieldName + ":{0 TO 3]", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='2']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='3']"); + + assertQ( + req("q", fieldName + ":[0 TO 3}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='2']"); + + assertQ( + req("q", fieldName + ":{0 TO 3}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='2']"); + + assertQ( + req("q", fieldName + ":{0 TO *}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='9']", + "0=count(//result/doc/" + type + "[@name='" + fieldName + "'][.='0'])", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1']"); + + assertQ( + req("q", fieldName + ":{* TO 3}", "fl", "id, " + fieldName, "sort", "id desc"), + "//*[@numFound='3']", + "0=count(//result/doc/" + type + "[@name='" + fieldName + "'][.='3'])", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='2']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='0']"); + + assertQ( + req("q", fieldName + ":[* TO 3}", "fl", "id, " + fieldName, "sort", "id desc"), + "//*[@numFound='3']", + "0=count(//result/doc/" + type + "[@name='" + fieldName + "'][.='3'])", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='2']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='0']"); + + assertQ( + req("q", fieldName + ":[* TO *}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0']", + "//result/doc[10]/" + type + "[@name='" + fieldName + "'][.='9']"); + + assertQ( + req( + "q", + fieldName + ":[0 TO 1] OR " + fieldName + ":[8 TO 9]", + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='8']", + "//result/doc[4]/" + type + "[@name='" + fieldName + "'][.='9']"); + + assertQ( + req("q", fieldName + ":[0 TO 1] AND " + fieldName + ":[1 TO 2]", "fl", "id, " + fieldName), + "//*[@numFound='1']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1']"); + + assertQ( + req( + "q", + fieldName + ":[0 TO 1] AND NOT " + fieldName + ":[1 TO 2]", + "fl", + "id, " + fieldName), + "//*[@numFound='1']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0']"); + + clearIndex(); + assertU(commit()); + + String[] arr; + if (testLong) { + arr = toAscendingStringArray(getRandomLongs(100, false), true); + } else { + arr = toAscendingStringArray(getRandomInts(100, false), true); + } + for (int i = 0; i < arr.length; i++) { + assertU(adoc("id", String.valueOf(i), fieldName, arr[i])); + } + assertU(commit()); + for (int i = 0; i < arr.length; i++) { + assertQ( + req("q", fieldName + ":[" + arr[0] + " TO " + arr[i] + "]", "fl", "id, " + fieldName), + "//*[@numFound='" + (i + 1) + "']"); + assertQ( + req("q", fieldName + ":{" + arr[0] + " TO " + arr[i] + "}", "fl", "id, " + fieldName), + "//*[@numFound='" + (Math.max(0, i - 1)) + "']"); + assertQ( + req( + "q", + fieldName + + ":[" + + arr[0] + + " TO " + + arr[i] + + "] AND " + + fieldName + + ":" + + arr[0].replace("-", "\\-"), + "fl", + "id, " + fieldName), + "//*[@numFound='1']"); + } + if (testLong) { + assertQ( + req( + "q", + fieldName + ":[" + Long.MIN_VALUE + " TO " + Long.MIN_VALUE + "}", + "fl", + "id, " + fieldName), + "//*[@numFound='0']"); + assertQ( + req( + "q", + fieldName + ":{" + Long.MAX_VALUE + " TO " + Long.MAX_VALUE + "]", + "fl", + "id, " + fieldName), + "//*[@numFound='0']"); + } else { + assertQ( + req( + "q", + fieldName + ":[" + Integer.MIN_VALUE + " TO " + Integer.MIN_VALUE + "}", + "fl", + "id, " + fieldName), + "//*[@numFound='0']"); + assertQ( + req( + "q", + fieldName + ":{" + Integer.MAX_VALUE + " TO " + Integer.MAX_VALUE + "]", + "fl", + "id, " + fieldName), + "//*[@numFound='0']"); + } + } + + private void doTestFieldFacetField( + String nonDocValuesField, String docValuesField, String[] numbers) { + assertTrue(numbers != null && numbers.length == 10); + + assertFalse(h.getCore().getLatestSchema().getField(docValuesField).multiValued()); + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + + for (int i = 0; i < 10; i++) { + assertU( + adoc("id", String.valueOf(i), docValuesField, numbers[i], nonDocValuesField, numbers[i])); + } + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + docValuesField, + "facet", + "true", + "facet.field", + docValuesField), + "//*[@numFound='10']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + docValuesField + + "']/int[@name='" + + numbers[1] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + docValuesField + + "']/int[@name='" + + numbers[2] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + docValuesField + + "']/int[@name='" + + numbers[3] + + "'][.='1']"); + + assertU(adoc("id", "10", docValuesField, numbers[1], nonDocValuesField, numbers[1])); + + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + docValuesField, + "facet", + "true", + "facet.field", + docValuesField), + "//*[@numFound='11']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + docValuesField + + "']/int[@name='" + + numbers[1] + + "'][.='2']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + docValuesField + + "']/int[@name='" + + numbers[2] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + docValuesField + + "']/int[@name='" + + numbers[3] + + "'][.='1']"); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + assertQEx( + "Expecting Exception", + "Can't facet on a NumericField without docValues", + req( + "q", + "*:*", + "fl", + "id, " + nonDocValuesField, + "facet", + "true", + "facet.field", + nonDocValuesField), + SolrException.ErrorCode.BAD_REQUEST); + } + + private void doTestIntFieldFunctionQuery(String field) { + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + int numVals = 10 * RANDOM_MULTIPLIER; + List values = getRandomInts(numVals, false); + String assertNumFound = "//*[@numFound='" + numVals + "']"; + String[] idAscXpathChecks = new String[numVals + 1]; + String[] idAscNegXpathChecks = new String[numVals + 1]; + idAscXpathChecks[0] = assertNumFound; + idAscNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < values.size(); ++i) { + assertU( + adoc( + "id", + Character.valueOf((char) ('A' + i)).toString(), + field, + String.valueOf(values.get(i)))); + // reminder: xpath array indexes start at 1 + idAscXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/int[@name='field(" + + field + + ")'][.='" + + values.get(i) + + "']"; + idAscNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/float[@name='product(-1," + + field + + ")'][.='" + + (-1.0f * (float) values.get(i)) + + "']"; + } + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", field(" + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscXpathChecks); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", product(-1," + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscNegXpathChecks); + + List> ascNegPosVals = + toAscendingPosVals(values.stream().map(v -> -v).collect(Collectors.toList()), true); + String[] ascNegXpathChecks = new String[numVals + 1]; + ascNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < ascNegPosVals.size(); ++i) { + PosVal posVal = ascNegPosVals.get(i); + ascNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/int[@name='" + + field + + "'][.='" + + values.get(posVal.pos) + + "']"; + } + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field, + "rows", + String.valueOf(numVals), + "sort", + "product(-1," + field + ") asc"), + ascNegXpathChecks); + + clearIndex(); + assertU(commit()); + } + + private void doTestLongFieldFunctionQuery(String field) { + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + int numVals = 10 * RANDOM_MULTIPLIER; + List values = getRandomLongs(numVals, false); + String assertNumFound = "//*[@numFound='" + numVals + "']"; + String[] idAscXpathChecks = new String[numVals + 1]; + String[] idAscNegXpathChecks = new String[numVals + 1]; + idAscXpathChecks[0] = assertNumFound; + idAscNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < values.size(); ++i) { + assertU( + adoc( + "id", + Character.valueOf((char) ('A' + i)).toString(), + field, + String.valueOf(values.get(i)))); + // reminder: xpath array indexes start at 1 + idAscXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/long[@name='field(" + + field + + ")'][.='" + + values.get(i) + + "']"; + idAscNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/float[@name='product(-1," + + field + + ")'][.='" + + (-1.0f * (float) values.get(i)) + + "']"; + } + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", field(" + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscXpathChecks); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", product(-1," + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscNegXpathChecks); + + List> ascNegPosVals = + toAscendingPosVals(values.stream().map(v -> -v).collect(Collectors.toList()), true); + String[] ascNegXpathChecks = new String[numVals + 1]; + ascNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < ascNegPosVals.size(); ++i) { + PosVal posVal = ascNegPosVals.get(i); + ascNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/long[@name='" + + field + + "'][.='" + + values.get(posVal.pos) + + "']"; + } + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field, + "rows", + String.valueOf(numVals), + "sort", + "product(-1," + field + ") asc"), + ascNegXpathChecks); + + clearIndex(); + assertU(commit()); + } + + /** + * Checks that the specified field can not be used as a value source, even if there are documents + * with (all) the specified values in the index. + * + * @param field the field name to try and sort on + * @param errSubStr substring to look for in the error msg + * @param values one or more values to put into the doc(s) in the index - may be more than one for + * multivalued fields + */ + private void doTestFieldFunctionQueryError(String field, String errSubStr, String... values) { + final int numDocs = atLeast(random(), 10); + for (int i = 0; i < numDocs; i++) { + SolrInputDocument doc = sdoc("id", String.valueOf(i)); + for (String v : values) { + doc.addField(field, v); + } + assertU(adoc(doc)); + } + + assertQEx( + "Should not be able to use field in function: " + field, + errSubStr, + req("q", "*:*", "fl", "id", "fq", "{!frange l=0 h=100}product(-1, " + field + ")"), + SolrException.ErrorCode.BAD_REQUEST); + + clearIndex(); + assertU(commit()); + + // empty index should (also) give same error + assertQEx( + "Should not be able to use field in function: " + field, + errSubStr, + req("q", "*:*", "fl", "id", "fq", "{!frange l=0 h=100}product(-1, " + field + ")"), + SolrException.ErrorCode.BAD_REQUEST); + } + + private void doTestFieldStats( + String field, + String dvField, + String[] numbers, + double min, + double max, + int count, + int missing, + double delta) { + String minMin = String.valueOf(min - Math.abs(delta * min)); + String maxMin = String.valueOf(min + Math.abs(delta * min)); + String minMax = String.valueOf(max - Math.abs(delta * max)); + String maxMax = String.valueOf(max + Math.abs(delta * max)); + for (int i = 0; i < numbers.length; i++) { + assertU(adoc("id", String.valueOf(i), dvField, numbers[i], field, numbers[i])); + } + assertU(adoc("id", String.valueOf(numbers.length))); + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(dvField).hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField(dvField).getType() instanceof NumericField); + assertQ( + req("q", "*:*", "fl", "id, " + dvField, "stats", "true", "stats.field", dvField), + "//*[@numFound='" + (numbers.length + 1) + "']", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/double[@name='min'][.>=" + + minMin + + "]", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/double[@name='min'][.<=" + + maxMin + + "]", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/double[@name='max'][.>=" + + minMax + + "]", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/double[@name='max'][.<=" + + maxMax + + "]", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/long[@name='count'][.='" + + count + + "']", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/long[@name='missing'][.='" + + missing + + "']"); + + assertFalse(h.getCore().getLatestSchema().getField(field).hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + assertQEx( + "Expecting Exception", + "Can't calculate stats on a NumericField without docValues", + req("q", "*:*", "fl", "id, " + field, "stats", "true", "stats.field", field), + SolrException.ErrorCode.BAD_REQUEST); + } + + private void doTestFieldMultiValuedExactQuery(final String fieldName, final String[] numbers) + throws Exception { + doTestFieldMultiValuedExactQuery(fieldName, numbers, true); + } + + /** + * @param fieldName the field to use for indexing and searching against + * @param numbers list of 20 values to index in 10 docs (pairwise) + * @param searchable set to true if searches against "field" should succeed, false if field is + * only stored and searches should always get numFound=0 + */ + private void doTestFieldMultiValuedExactQuery( + final String fieldName, final String[] numbers, final boolean searchable) { + + final String MATCH_ONE = "//*[@numFound='" + (searchable ? "1" : "0") + "']"; + final String MATCH_TWO = "//*[@numFound='" + (searchable ? "2" : "0") + "']"; + + assertTrue(numbers != null && numbers.length == 20); + assertTrue(h.getCore().getLatestSchema().getField(fieldName).multiValued()); + assertTrue(h.getCore().getLatestSchema().getField(fieldName).getType() instanceof NumericField); + for (int i = 0; i < 10; i++) { + assertU(adoc("id", String.valueOf(i), fieldName, numbers[i], fieldName, numbers[i + 10])); + } + assertU(commit()); + FieldType type = h.getCore().getLatestSchema().getField(fieldName).getType(); + for (int i = 0; i < 20; i++) { + if (type instanceof DateField) { + assertQ(req("q", fieldName + ":\"" + numbers[i] + "\""), MATCH_ONE); + } else { + assertQ(req("q", fieldName + ":" + numbers[i].replace("-", "\\-")), MATCH_ONE); + } + } + + for (int i = 0; i < 20; i++) { + if (type instanceof DateField) { + assertQ( + req( + "q", + fieldName + + ":\"" + + numbers[i] + + "\"" + + " OR " + + fieldName + + ":\"" + + numbers[(i + 1) % 10] + + "\""), + MATCH_TWO); + } else { + assertQ( + req( + "q", + fieldName + + ":" + + numbers[i].replace("-", "\\-") + + " OR " + + fieldName + + ":" + + numbers[(i + 1) % 10].replace("-", "\\-")), + MATCH_TWO); + } + } + } + + private void doTestFieldMultiValuedReturn(String fieldName, String type, String[] numbers) { + assertTrue(numbers != null && numbers.length == 20); + assertTrue(h.getCore().getLatestSchema().getField(fieldName).multiValued()); + assertTrue(h.getCore().getLatestSchema().getField(fieldName).getType() instanceof NumericField); + for (int i = 9; i >= 0; i--) { + assertU(adoc("id", String.valueOf(i), fieldName, numbers[i], fieldName, numbers[i + 10])); + } + // Check using RTG before commit + if (Boolean.getBoolean("solr.index.updatelog.enabled")) { + for (int i = 0; i < 10; i++) { + assertQ( + req("qt", "/get", "id", String.valueOf(i)), + "//doc/arr[@name='" + fieldName + "']/" + type + "[.='" + numbers[i] + "']", + "//doc/arr[@name='" + fieldName + "']/" + type + "[.='" + numbers[i + 10] + "']", + "count(//doc/arr[@name='" + fieldName + "']/" + type + ")=2"); + } + } + // Check using RTG after commit + assertU(commit()); + if (Boolean.getBoolean("solr.index.updatelog.enabled")) { + for (int i = 0; i < 10; i++) { + assertQ( + req("qt", "/get", "id", String.valueOf(i)), + "//doc/arr[@name='" + fieldName + "']/" + type + "[.='" + numbers[i] + "']", + "//doc/arr[@name='" + fieldName + "']/" + type + "[.='" + numbers[i + 10] + "']", + "count(//doc/arr[@name='" + fieldName + "']/" + type + ")=2"); + } + } + String[] expected = new String[21]; + expected[0] = "//*[@numFound='10']"; + for (int i = 1; i <= 10; i++) { + // checks for each doc's two values aren't next to each other in array, but that doesn't + // matter + // for correctness + expected[i] = + "//result/doc[" + + i + + "]/arr[@name='" + + fieldName + + "']/" + + type + + "[.='" + + numbers[i - 1] + + "']"; + expected[i + 10] = + "//result/doc[" + + i + + "]/arr[@name='" + + fieldName + + "']/" + + type + + "[.='" + + numbers[i + 9] + + "']"; + } + assertQ(req("q", "*:*", "fl", "id, " + fieldName, "sort", "id asc"), expected); + } + + private void doTestFieldMultiValuedRangeQuery(String fieldName, String type, String[] numbers) { + assertTrue(numbers != null && numbers.length == 20); + SchemaField sf = h.getCore().getLatestSchema().getField(fieldName); + assertTrue(sf.multiValued()); + assertTrue(sf.getType() instanceof NumericField); + for (int i = 9; i >= 0; i--) { + assertU(adoc("id", String.valueOf(i), fieldName, numbers[i], fieldName, numbers[i + 10])); + } + assertU(commit()); + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:[%s TO %s]", fieldName, numbers[0], numbers[3]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[2][.='" + numbers[10] + "']", + "//result/doc[2]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[1] + "']", + "//result/doc[2]/arr[@name='" + fieldName + "']/" + type + "[2][.='" + numbers[11] + "']", + "//result/doc[3]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[2] + "']", + "//result/doc[3]/arr[@name='" + fieldName + "']/" + type + "[2][.='" + numbers[12] + "']", + "//result/doc[4]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[3] + "']", + "//result/doc[4]/arr[@name='" + fieldName + "']/" + type + "[2][.='" + numbers[13] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:{%s TO %s]", fieldName, numbers[0], numbers[3]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[1] + "']", + "//result/doc[2]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[2] + "']", + "//result/doc[3]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[3] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:[%s TO %s}", fieldName, numbers[0], numbers[3]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']", + "//result/doc[2]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[1] + "']", + "//result/doc[3]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[2] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:{%s TO %s}", fieldName, numbers[0], numbers[3]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[1] + "']", + "//result/doc[2]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[2] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:{%s TO *}", fieldName, numbers[0]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:{%s TO *}", fieldName, numbers[10]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='9']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[1] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:{* TO %s}", fieldName, numbers[3]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']"); + + assertQ( + req( + "q", + String.format(Locale.ROOT, "%s:[* TO %s}", fieldName, numbers[3]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']"); + + assertQ( + req("q", fieldName + ":[* TO *}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']", + "//result/doc[10]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[9] + "']"); + + assertQ( + req( + "q", + String.format( + Locale.ROOT, + "%s:[%s TO %s] OR %s:[%s TO %s]", + fieldName, + numbers[0], + numbers[1], + fieldName, + numbers[8], + numbers[9]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']", + "//result/doc[2]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[1] + "']", + "//result/doc[3]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[8] + "']", + "//result/doc[4]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[9] + "']"); + + assertQ( + req( + "q", + String.format( + Locale.ROOT, + "%s:[%s TO %s] OR %s:[%s TO %s]", + fieldName, + numbers[0], + numbers[0], + fieldName, + numbers[10], + numbers[10]), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='1']", + "//result/doc[1]/arr[@name='" + fieldName + "']/" + type + "[1][.='" + numbers[0] + "']"); + + if (sf.getType().getNumberType() == NumberType.FLOAT + || sf.getType().getNumberType() == NumberType.DOUBLE) { + doTestDoubleFloatRangeLimits(fieldName, sf.getType().getNumberType() == NumberType.DOUBLE); + } + } + + private void doTestFieldMultiValuedFacetField( + String nonDocValuesField, String dvFieldName, String[] numbers) { + assertTrue(numbers != null && numbers.length == 20); + assertTrue(h.getCore().getLatestSchema().getField(dvFieldName).multiValued()); + assertTrue(h.getCore().getLatestSchema().getField(dvFieldName).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(dvFieldName).getType() instanceof NumericField); + + for (int i = 0; i < 10; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + dvFieldName, + numbers[i], + dvFieldName, + numbers[i + 10], + nonDocValuesField, + numbers[i], + nonDocValuesField, + numbers[i + 10])); + if (rarely()) { + assertU(commit()); + } + } + assertU(commit()); + + assertQ( + req("q", "*:*", "fl", "id, " + dvFieldName, "facet", "true", "facet.field", dvFieldName), + "//*[@numFound='10']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[1] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[2] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[3] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[10] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[11] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[12] + + "'][.='1']"); + + assertU(adoc("id", "10", dvFieldName, numbers[1], nonDocValuesField, numbers[1])); + + assertU(commit()); + assertQ( + req("q", "*:*", "fl", "id, " + dvFieldName, "facet", "true", "facet.field", dvFieldName), + "//*[@numFound='11']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[1] + + "'][.='2']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[2] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[3] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[10] + + "'][.='1']"); + + assertU( + adoc( + "id", + "10", + dvFieldName, + numbers[1], + nonDocValuesField, + numbers[1], + dvFieldName, + numbers[1], + nonDocValuesField, + numbers[1])); + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + dvFieldName, + "facet", + "true", + "facet.field", + dvFieldName, + "facet.missing", + "true"), + "//*[@numFound='11']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[1] + + "'][.='2']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[2] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[3] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[10] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[not(@name)][.='0']"); + + assertU(adoc("id", "10")); // add missing values + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + dvFieldName, + "facet", + "true", + "facet.field", + dvFieldName, + "facet.missing", + "true"), + "//*[@numFound='11']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[1] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[2] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[3] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[10] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[not(@name)][.='1']"); + + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + dvFieldName, + "facet", + "true", + "facet.field", + dvFieldName, + "facet.mincount", + "3"), + "//*[@numFound='11']", + "count(//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int)=0"); + + assertQ( + req("q", "id:0", "fl", "id, " + dvFieldName, "facet", "true", "facet.field", dvFieldName), + "//*[@numFound='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[0] + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + numbers[10] + + "'][.='1']", + "count(//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int)=2"); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + assertQEx( + "Expecting Exception", + "Can't facet on a NumericField without docValues", + req( + "q", + "*:*", + "fl", + "id, " + nonDocValuesField, + "facet", + "true", + "facet.field", + nonDocValuesField), + SolrException.ErrorCode.BAD_REQUEST); + clearIndex(); + assertU(commit()); + + String smaller, larger; + try { + if (Long.parseLong(numbers[1]) < Long.parseLong(numbers[2])) { + smaller = numbers[1]; + larger = numbers[2]; + } else { + smaller = numbers[2]; + larger = numbers[1]; + } + } catch (NumberFormatException e) { + try { + if (Double.parseDouble(numbers[1]) < Double.parseDouble(numbers[2])) { + smaller = numbers[1]; + larger = numbers[2]; + } else { + smaller = numbers[2]; + larger = numbers[1]; + } + } catch (NumberFormatException e2) { + if (DateMathParser.parseMath(null, numbers[1]).getTime() + < DateMathParser.parseMath(null, numbers[2]).getTime()) { + smaller = numbers[1]; + larger = numbers[2]; + } else { + smaller = numbers[2]; + larger = numbers[1]; + } + } + } + + assertU(adoc("id", "1", dvFieldName, smaller, dvFieldName, larger)); + assertU(adoc("id", "2", dvFieldName, larger)); + assertU(commit()); + + assertQ( + req("q", "*:*", "fl", "id, " + dvFieldName, "facet", "true", "facet.field", dvFieldName), + "//*[@numFound='2']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + larger + + "'][.='2']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + smaller + + "'][.='1']", + "count(//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int)=2"); + + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + dvFieldName, + "facet", + "true", + "facet.field", + dvFieldName, + "facet.sort", + "index"), + "//*[@numFound='2']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + smaller + + "'][.='1']", + "//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int[@name='" + + larger + + "'][.='2']", + "count(//lst[@name='facet_counts']/lst[@name='facet_fields']/lst[@name='" + + dvFieldName + + "']/int)=2"); + + clearIndex(); + assertU(commit()); + } + + private void doTestFieldMultiValuedFunctionQuery( + String nonDocValuesField, String docValuesField, String[] numbers) { + assertTrue(numbers != null && numbers.length == 20); + for (int i = 0; i < 10; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + docValuesField, + numbers[i], + docValuesField, + numbers[i + 10], + nonDocValuesField, + numbers[i], + nonDocValuesField, + numbers[i + 10])); + } + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField(docValuesField).multiValued()); + assertTrue( + h.getCore().getLatestSchema().getField(docValuesField).getType() instanceof NumericField); + String function = "field(" + docValuesField + ", min)"; + + assertQ( + req("q", "*:*", "fl", "id, " + docValuesField, "sort", function + " desc"), + "//*[@numFound='10']", + "//result/doc[1]/str[@name='id'][.='9']", + "//result/doc[2]/str[@name='id'][.='8']", + "//result/doc[3]/str[@name='id'][.='7']", + "//result/doc[10]/str[@name='id'][.='0']"); + + assertFalse(h.getCore().getLatestSchema().getField(nonDocValuesField).hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField(nonDocValuesField).multiValued()); + assertTrue( + h.getCore().getLatestSchema().getField(nonDocValuesField).getType() + instanceof NumericField); + + function = "field(" + nonDocValuesField + ",min)"; + + assertQEx( + "Expecting Exception", + "sort param could not be parsed as a query", + req("q", "*:*", "fl", "id", "sort", function + " desc"), + SolrException.ErrorCode.BAD_REQUEST); + + assertQEx( + "Expecting Exception", + "docValues='true' is required to select 'min' value from multivalued field (" + + nonDocValuesField + + ") at query time", + req("q", "*:*", "fl", "id, " + function), + SolrException.ErrorCode.BAD_REQUEST); + + function = "field(" + docValuesField + ",foo)"; + assertQEx( + "Expecting Exception", + "Multi-Valued field selector 'foo' not supported", + req("q", "*:*", "fl", "id, " + function), + SolrException.ErrorCode.BAD_REQUEST); + } + + private void doTestMultiValuedFieldsAtomicUpdates(String field, String type, String[] values) { + assertEquals(3, values.length); + assertU(adoc(sdoc("id", "1", field, String.valueOf(values[0])))); + assertU(commit()); + + assertQ( + req("q", "id:1"), + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[0] + "']", + "count(//result/doc[1]/arr[@name='" + field + "']/" + type + ")=1"); + + assertU(adoc(sdoc("id", "1", field, Map.of("add", values[1])))); + assertU(commit()); + + assertQ( + req("q", "id:1"), + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[0] + "']", + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[1] + "']", + "count(//result/doc[1]/arr[@name='" + field + "']/" + type + ")=2"); + + assertU(adoc(sdoc("id", "1", field, Map.of("remove", values[0])))); + assertU(commit()); + + assertQ( + req("q", "id:1"), + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[1] + "']", + "count(//result/doc[1]/arr[@name='" + field + "']/" + type + ")=1"); + + assertU(adoc(sdoc("id", "1", field, Map.of("set", Arrays.asList(values))))); + assertU(commit()); + + assertQ( + req("q", "id:1"), + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[0] + "']", + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[1] + "']", + "//result/doc[1]/arr[@name='" + field + "']/" + type + "[.='" + values[2] + "']", + "count(//result/doc[1]/arr[@name='" + field + "']/" + type + ")=3"); + + assertU(adoc(sdoc("id", "1", field, Map.of("removeregex", ".*")))); + assertU(commit()); + + assertQ(req("q", "id:1"), "count(//result/doc[1]/arr[@name='" + field + "']/" + type + ")=0"); + } + + private void doTestIntFieldsAtomicUpdates(String field) { + int number1 = random().nextInt(); + int number2; + long inc1; + for (; ; ) { + number2 = random().nextInt(); + inc1 = (long) number2 - number1; + if (Math.abs(inc1) < (long) Integer.MAX_VALUE) { + break; + } + } + assertU(adoc(sdoc("id", "1", field, String.valueOf(number1)))); + assertU(commit()); + + assertU(adoc(sdoc("id", "1", field, Map.of("inc", (int) inc1)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/int[@name='" + field + "'][.='" + number2 + "']"); + + int number3 = random().nextInt(); + assertU(adoc(sdoc("id", "1", field, Map.of("set", number3)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/int[@name='" + field + "'][.='" + number3 + "']"); + } + + private void doTestLongFieldsAtomicUpdates(String field) { + long number1 = random().nextLong(); + long number2; + BigInteger inc1; + for (; ; ) { + number2 = random().nextLong(); + inc1 = BigInteger.valueOf(number2).subtract(BigInteger.valueOf(number1)); + if (inc1.abs().compareTo(BigInteger.valueOf(Long.MAX_VALUE)) <= 0) { + break; + } + } + assertU(adoc(sdoc("id", "1", field, String.valueOf(number1)))); + assertU(commit()); + + assertU(adoc(sdoc("id", "1", field, Map.of("inc", inc1.longValueExact())))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/long[@name='" + field + "'][.='" + number2 + "']"); + + long number3 = random().nextLong(); + assertU(adoc(sdoc("id", "1", field, Map.of("set", number3)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/long[@name='" + field + "'][.='" + number3 + "']"); + } + + private void doTestFloatFieldExactQuery(final String field, boolean testDouble) throws Exception { + doTestFloatFieldExactQuery(field, true, testDouble); + } + + /** + * @param field the field to use for indexing and searching against + * @param searchable set to true if searches against "field" should succeed, false if field is + * only stored and searches should always get numFound=0 + */ + private void doTestFloatFieldExactQuery( + final String field, final boolean searchable, final boolean testDouble) { + int numValues = 10 * RANDOM_MULTIPLIER; + Map randCount = new HashMap<>(numValues); + String[] rand = + testDouble + ? toStringArray(getRandomDoubles(numValues, false)) + : toStringArray(getRandomFloats(numValues, false)); + for (int i = 0; i < numValues; i++) { + randCount.merge(rand[i], 1, (a, b) -> a + b); // count unique values + assertU(adoc("id", String.valueOf(i), field, rand[i])); + } + assertU(commit()); + + for (int i = 0; i < numValues; i++) { + assertQ( + req( + "q", + field + ":" + (rand[i].startsWith("-") ? "\\" : "") + rand[i], + "fl", + "id," + field), + getTestString(searchable, randCount.get(rand[i]))); + } + + StringBuilder builder = new StringBuilder(); + for (String value : randCount.keySet()) { + if (builder.length() != 0) { + builder.append(" OR "); + } + if (value.startsWith("-")) { + builder.append("\\"); // escape negative sign + } + builder.append(value); + } + assertQ( + req("debug", "true", "q", field + ":(" + builder + ")"), + getTestString(searchable, numValues)); + + clearIndex(); + assertU(commit()); + } + + /** + * For each value, creates a doc with that value in the specified field and then asserts that + * asc/desc sorts on that field succeeds and that the docs are in the (relatively) expected order + * + * @param field name of field to sort on + * @param values list of values in ascending order + */ + private > void doTestFieldSort(String field, List values) { + assertTrue(values != null && 2 <= values.size()); + + final List docs = new ArrayList<>(values.size()); + final String[] ascXpathChecks = new String[values.size() + 1]; + final String[] descXpathChecks = new String[values.size() + 1]; + ascXpathChecks[values.size()] = "//*[@numFound='" + values.size() + "']"; + descXpathChecks[values.size()] = "//*[@numFound='" + values.size() + "']"; + + boolean missingFirst = field.endsWith("_sml") == false; + + List> ascendingPosVals = toAscendingPosVals(values, missingFirst); + for (int i = ascendingPosVals.size() - 1; i >= 0; --i) { + T value = ascendingPosVals.get(i).val; + if (value == null) { + docs.add(sdoc("id", String.valueOf(i))); // null => missing value + } else { + docs.add(sdoc("id", String.valueOf(i), field, String.valueOf(value))); + } + // reminder: xpath array indexes start at 1 + ascXpathChecks[i] = "//result/doc[" + (1 + i) + "]/str[@name='id'][.='" + i + "']"; + } + List> descendingPosVals = + toDescendingPosVals( + ascendingPosVals.stream().map(pv -> pv.val).collect(Collectors.toList()), missingFirst); + for (int i = descendingPosVals.size() - 1; i >= 0; --i) { + descXpathChecks[i] = + "//result/doc[" + (i + 1) + "]/str[@name='id'][.='" + descendingPosVals.get(i).pos + "']"; + } + + // ensure doc add order doesn't affect results + Collections.shuffle(docs, random()); + for (SolrInputDocument doc : docs) { + assertU(adoc(doc)); + } + assertU(commit()); + + assertQ(req("q", "*:*", "fl", "id, " + field, "sort", field + " asc, id asc"), ascXpathChecks); + assertQ( + req("q", "*:*", "fl", "id, " + field, "sort", field + " desc, id desc"), descXpathChecks); + + clearIndex(); + assertU(commit()); + } + + /** + * Checks that the specified field can not be sorted on, even if there are documents with (all) + * the specified values in the index. + * + * @param field the field name to try and sort on + * @param errSubStr substring to look for in the error msg + * @param values one or more values to put into the doc(s) in the index - may be more than one for + * multivalued fields + */ + private void doTestFieldSortError(String field, String errSubStr, String... values) { + + final int numDocs = atLeast(random(), 10); + for (int i = 0; i < numDocs; i++) { + SolrInputDocument doc = sdoc("id", String.valueOf(i)); + for (String v : values) { + doc.addField(field, v); + } + assertU(adoc(doc)); + } + + assertQEx( + "Should not be able to sort on field: " + field, + errSubStr, + req("q", "*:*", "fl", "id", "sort", field + " desc"), + SolrException.ErrorCode.BAD_REQUEST); + + clearIndex(); + assertU(commit()); + + // empty index should (also) give same error + assertQEx( + "Should not be able to sort on field: " + field, + errSubStr, + req("q", "*:*", "fl", "id", "sort", field + " desc"), + SolrException.ErrorCode.BAD_REQUEST); + } + + private void doTestFloatFieldRangeQuery(String fieldName, String type, boolean testDouble) { + for (int i = 9; i >= 0; i--) { + assertU(adoc("id", String.valueOf(i), fieldName, String.valueOf(i))); + } + assertU(commit()); + assertQ( + req("q", fieldName + ":[0 TO 3]", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0.0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1.0']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='2.0']", + "//result/doc[4]/" + type + "[@name='" + fieldName + "'][.='3.0']"); + + assertQ( + req("q", fieldName + ":{0 TO 3]", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1.0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='2.0']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='3.0']"); + + assertQ( + req("q", fieldName + ":[0 TO 3}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0.0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1.0']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='2.0']"); + + assertQ( + req("q", fieldName + ":{0 TO 3}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1.0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='2.0']"); + + assertQ( + req("q", fieldName + ":{0 TO *}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='9']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1.0']"); + + assertQ( + req("q", fieldName + ":{* TO 3}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0.0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1.0']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='2.0']"); + + assertQ( + req("q", fieldName + ":[* TO 3}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0.0']", + "//result/doc[2]/" + type + "[@name='" + fieldName + "'][.='1.0']", + "//result/doc[3]/" + type + "[@name='" + fieldName + "'][.='2.0']"); + + assertQ( + req("q", fieldName + ":[* TO *}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='0.0']", + "//result/doc[10]/" + type + "[@name='" + fieldName + "'][.='9.0']"); + + assertQ( + req("q", fieldName + ":[0.9 TO 1.01]", "fl", "id, " + fieldName), + "//*[@numFound='1']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1.0']"); + + assertQ( + req("q", fieldName + ":{0.9 TO 1.01}", "fl", "id, " + fieldName), + "//*[@numFound='1']", + "//result/doc[1]/" + type + "[@name='" + fieldName + "'][.='1.0']"); + + clearIndex(); + assertU(commit()); + + String[] arr; + if (testDouble) { + arr = toAscendingStringArray(getRandomDoubles(10, false), true); + } else { + arr = toAscendingStringArray(getRandomFloats(10, false), true); + } + for (int i = 0; i < arr.length; i++) { + assertU(adoc("id", String.valueOf(i), fieldName, arr[i])); + } + assertU(commit()); + for (int i = 0; i < arr.length; i++) { + assertQ( + req("q", fieldName + ":[" + arr[0] + " TO " + arr[i] + "]", "fl", "id, " + fieldName), + "//*[@numFound='" + (i + 1) + "']"); + assertQ( + req("q", fieldName + ":{" + arr[0] + " TO " + arr[i] + "}", "fl", "id, " + fieldName), + "//*[@numFound='" + (Math.max(0, i - 1)) + "']"); + } + doTestDoubleFloatRangeLimits(fieldName, testDouble); + } + + private void doTestDoubleFloatRangeLimits(String fieldName, boolean testDouble) { + // POSITIVE/NEGATIVE_INFINITY toString is the same for Double and Float, it's OK to use this + // code for both cases + String positiveInfinity = String.valueOf(Double.POSITIVE_INFINITY); + String negativeInfinity = String.valueOf(Double.NEGATIVE_INFINITY); + String minVal = String.valueOf(testDouble ? Double.MIN_VALUE : Float.MIN_VALUE); + String maxVal = String.valueOf(testDouble ? Double.MAX_VALUE : Float.MAX_VALUE); + String negativeMinVal = "-" + minVal; + String negativeMaxVal = "-" + maxVal; + clearIndex(); + assertU(adoc("id", "1", fieldName, minVal)); + assertU(adoc("id", "2", fieldName, maxVal)); + assertU(adoc("id", "3", fieldName, negativeInfinity)); + assertU(adoc("id", "4", fieldName, positiveInfinity)); + assertU(adoc("id", "5", fieldName, negativeMinVal)); + assertU(adoc("id", "6", fieldName, negativeMaxVal)); + assertU(commit()); + // negative to negative + assertAllInclusiveExclusiveVariations(fieldName, "*", "-1", 2, 2, 2, 2); + assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "-1", 1, 2, 1, 2); + assertAllInclusiveExclusiveVariations(fieldName, negativeMaxVal, negativeMinVal, 0, 1, 1, 2); + // negative to cero + assertAllInclusiveExclusiveVariations(fieldName, "*", "-0.0f", 3, 3, 3, 3); + assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "-0.0f", 2, 3, 2, 3); + assertAllInclusiveExclusiveVariations(fieldName, negativeMinVal, "-0.0f", 0, 1, 0, 1); + + assertAllInclusiveExclusiveVariations(fieldName, "*", "0", 3, 3, 3, 3); + assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "0", 2, 3, 2, 3); + assertAllInclusiveExclusiveVariations(fieldName, negativeMinVal, "0", 0, 1, 0, 1); + // negative to positive + assertAllInclusiveExclusiveVariations(fieldName, "*", "1", 4, 4, 4, 4); + assertAllInclusiveExclusiveVariations(fieldName, "-1", "*", 4, 4, 4, 4); + assertAllInclusiveExclusiveVariations(fieldName, "-1", "1", 2, 2, 2, 2); + assertAllInclusiveExclusiveVariations(fieldName, "*", "*", 6, 6, 6, 6); + + assertAllInclusiveExclusiveVariations(fieldName, "-1", positiveInfinity, 3, 3, 4, 4); + assertAllInclusiveExclusiveVariations(fieldName, negativeInfinity, "1", 3, 4, 3, 4); + assertAllInclusiveExclusiveVariations( + fieldName, negativeInfinity, positiveInfinity, 4, 5, 5, 6); + + assertAllInclusiveExclusiveVariations(fieldName, negativeMinVal, minVal, 0, 1, 1, 2); + assertAllInclusiveExclusiveVariations(fieldName, negativeMaxVal, maxVal, 2, 3, 3, 4); + // cero to positive + assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", "*", 3, 3, 3, 3); + assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", positiveInfinity, 2, 2, 3, 3); + assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", minVal, 0, 0, 1, 1); + + assertAllInclusiveExclusiveVariations(fieldName, "0", "*", 3, 3, 3, 3); + assertAllInclusiveExclusiveVariations(fieldName, "0", positiveInfinity, 2, 2, 3, 3); + assertAllInclusiveExclusiveVariations(fieldName, "0", minVal, 0, 0, 1, 1); + // positive to positive + assertAllInclusiveExclusiveVariations(fieldName, "1", "*", 2, 2, 2, 2); + assertAllInclusiveExclusiveVariations(fieldName, "1", positiveInfinity, 1, 1, 2, 2); + assertAllInclusiveExclusiveVariations(fieldName, minVal, maxVal, 0, 1, 1, 2); + + // inverted limits + assertAllInclusiveExclusiveVariations(fieldName, "1", "-1", 0, 0, 0, 0); + assertAllInclusiveExclusiveVariations( + fieldName, positiveInfinity, negativeInfinity, 0, 0, 0, 0); + assertAllInclusiveExclusiveVariations(fieldName, minVal, negativeMinVal, 0, 0, 0, 0); + + // MatchNoDocs cases + assertAllInclusiveExclusiveVariations( + fieldName, negativeInfinity, negativeInfinity, 0, 0, 0, 1); + assertAllInclusiveExclusiveVariations( + fieldName, positiveInfinity, positiveInfinity, 0, 0, 0, 1); + + clearIndex(); + assertU(adoc("id", "1", fieldName, "0.0")); + assertU(adoc("id", "2", fieldName, "-0.0")); + assertU(commit()); + assertAllInclusiveExclusiveVariations(fieldName, "*", "*", 2, 2, 2, 2); + assertAllInclusiveExclusiveVariations(fieldName, "*", "0", 1, 1, 2, 2); + assertAllInclusiveExclusiveVariations(fieldName, "0", "*", 0, 1, 0, 1); + assertAllInclusiveExclusiveVariations(fieldName, "*", "-0.0f", 0, 0, 1, 1); + assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", "*", 1, 2, 1, 2); + assertAllInclusiveExclusiveVariations(fieldName, "-0.0f", "0", 0, 1, 1, 2); + } + + private void assertAllInclusiveExclusiveVariations( + String fieldName, + String min, + String max, + int countExclusiveExclusive, + int countInclusiveExclusive, + int countExclusiveInclusive, + int countInclusiveInclusive) { + assertQ( + req("q", fieldName + ":{" + min + " TO " + max + "}", "fl", "id, " + fieldName), + "//*[@numFound='" + countExclusiveExclusive + "']"); + assertQ( + req("q", fieldName + ":[" + min + " TO " + max + "}", "fl", "id, " + fieldName), + "//*[@numFound='" + countInclusiveExclusive + "']"); + assertQ( + req("q", fieldName + ":{" + min + " TO " + max + "]", "fl", "id, " + fieldName), + "//*[@numFound='" + countExclusiveInclusive + "']"); + assertQ( + req("q", fieldName + ":[" + min + " TO " + max + "]", "fl", "id, " + fieldName), + "//*[@numFound='" + countInclusiveInclusive + "']"); + } + + private void doTestFloatFieldFunctionQuery(String field) { + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + int numVals = 10 * RANDOM_MULTIPLIER; + List values = getRandomFloats(numVals, false); + String assertNumFound = "//*[@numFound='" + numVals + "']"; + String[] idAscXpathChecks = new String[numVals + 1]; + String[] idAscNegXpathChecks = new String[numVals + 1]; + idAscXpathChecks[0] = assertNumFound; + idAscNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < values.size(); ++i) { + assertU( + adoc( + "id", + Character.valueOf((char) ('A' + i)).toString(), + field, + String.valueOf(values.get(i)))); + // reminder: xpath array indexes start at 1 + idAscXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/float[@name='field(" + + field + + ")'][.='" + + values.get(i) + + "']"; + idAscNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/float[@name='product(-1," + + field + + ")'][.='" + + (-1.0f * values.get(i)) + + "']"; + } + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", field(" + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscXpathChecks); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", product(-1," + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscNegXpathChecks); + + List> ascNegPosVals = + toAscendingPosVals(values.stream().map(v -> -v).collect(Collectors.toList()), true); + String[] ascNegXpathChecks = new String[numVals + 1]; + ascNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < ascNegPosVals.size(); ++i) { + PosVal posVal = ascNegPosVals.get(i); + ascNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/float[@name='" + + field + + "'][.='" + + values.get(posVal.pos) + + "']"; + } + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field, + "rows", + String.valueOf(numVals), + "sort", + "product(-1," + field + ") asc"), + ascNegXpathChecks); + + clearIndex(); + assertU(commit()); + } + + private void doTestDoubleFieldFunctionQuery(String field) { + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + int numVals = 10 * RANDOM_MULTIPLIER; + // Restrict values to float range; otherwise conversion to float will cause truncation -> + // undefined results + List values = + getRandomList( + numVals, + false, + () -> { + Float f = Float.NaN; + while (f.isNaN()) { + f = Float.intBitsToFloat(random().nextInt()); + } + return f.doubleValue(); + }); + String assertNumFound = "//*[@numFound='" + numVals + "']"; + String[] idAscXpathChecks = new String[numVals + 1]; + String[] idAscNegXpathChecks = new String[numVals + 1]; + idAscXpathChecks[0] = assertNumFound; + idAscNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < values.size(); ++i) { + assertU( + adoc( + "id", + Character.valueOf((char) ('A' + i)).toString(), + field, + String.valueOf(values.get(i)))); + // reminder: xpath array indexes start at 1 + idAscXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/double[@name='field(" + + field + + ")'][.='" + + values.get(i) + + "']"; + idAscNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/float[@name='product(-1," + + field + + ")'][.='" + + (-1.0f * values.get(i).floatValue()) + + "']"; + } + assertU(commit()); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", field(" + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscXpathChecks); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", product(-1," + field + ")", + "rows", + String.valueOf(numVals), + "sort", + "id asc"), + idAscNegXpathChecks); + + // Intentionally use floats here to mimic server-side function sorting + List> ascNegPosVals = + toAscendingPosVals( + values.stream().map(v -> -v.floatValue()).collect(Collectors.toList()), true); + String[] ascNegXpathChecks = new String[numVals + 1]; + ascNegXpathChecks[0] = assertNumFound; + for (int i = 0; i < ascNegPosVals.size(); ++i) { + PosVal posVal = ascNegPosVals.get(i); + ascNegXpathChecks[i + 1] = + "//result/doc[" + + (1 + i) + + "]/double[@name='" + + field + + "'][.='" + + values.get(posVal.pos) + + "']"; + } + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field, + "rows", + String.valueOf(numVals), + "sort", + "product(-1," + field + ") asc"), + ascNegXpathChecks); + + clearIndex(); + assertU(commit()); + } + + private void doTestSetQueries(String fieldName, String[] values, boolean multiValued) { + for (int i = 0; i < values.length; i++) { + assertU(adoc("id", String.valueOf(i), fieldName, values[i])); + } + assertU(commit()); + SchemaField sf = h.getCore().getLatestSchema().getField(fieldName); + assertTrue(sf.getType() instanceof NumericField); + + for (String value : values) { + assertQ( + req("q", "{!term f='" + fieldName + "'}" + value, "fl", "id," + fieldName), + "//*[@numFound='1']"); + } + + for (int i = 0; i < values.length; i++) { + assertQ( + req( + "q", + "{!terms f='" + fieldName + "'}" + values[i] + "," + values[(i + 1) % values.length], + "fl", + "id," + fieldName), + "//*[@numFound='2']"); + } + + assertTrue(values.length > SolrQueryParser.TERMS_QUERY_THRESHOLD); + int numTerms = SolrQueryParser.TERMS_QUERY_THRESHOLD + 1; + StringBuilder builder = new StringBuilder(fieldName + ":("); + for (int i = 0; i < numTerms; i++) { + if (sf.getType().getNumberType() == NumberType.DATE) { + builder.append(values[i].replaceAll("(:|^[-+])", "\\\\$1")).append(' '); + } else { + builder.append(String.valueOf(values[i]).replace("-", "\\-")).append(' '); + } + } + builder.append(')'); + assertQ( + req( + CommonParams.DEBUG, + CommonParams.QUERY, + "q", + "*:*", + "fq", + builder.toString(), + "fl", + "id," + fieldName), + "//*[@numFound='" + numTerms + "']", + "//*[@name='parsed_filter_queries']/str[" + + getSetQueryToString(fieldName, values, numTerms) + + "]"); + + if (multiValued) { + clearIndex(); + assertU(commit()); + for (int i = 0; i < values.length; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + fieldName, + values[i], + fieldName, + values[(i + 1) % values.length])); + } + assertU(commit()); + for (String value : values) { + assertQ( + req("q", "{!term f='" + fieldName + "'}" + value, "fl", "id," + fieldName), + "//*[@numFound='2']"); + } + + for (int i = 0; i < values.length; i++) { + assertQ( + req( + "q", + "{!terms f='" + + fieldName + + "'}" + + values[i] + + "," + + values[(i + 1) % values.length], + "fl", + "id," + fieldName), + "//*[@numFound='3']"); + } + } + } + + private String getSetQueryToString(String fieldName, String[] values, int numTerms) { + SchemaField sf = h.getCore().getLatestSchema().getField(fieldName); + List rules = new ArrayList<>(); + boolean isMultiQuery = false; + if (sf.indexed() && sf.hasDocValues()) { + isMultiQuery = true; + rules.add("starts-with(., 'IndexOrDocValuesQuery')"); + } + if (sf.enhancedIndex()) { + if (isMultiQuery) { + rules.add("contains(., 'indexQuery=" + fieldName + ":(')"); + } else { + rules.add("starts-with(., 'TermInSetQuery(" + fieldName + ":(')"); + } + } else if (sf.indexed()) { + if (isMultiQuery) { + rules.add("contains(., 'indexQuery=" + fieldName + ":{')"); + } else { + rules.add("starts-with(., '(" + fieldName + ":{')"); + } + } + if (sf.hasDocValues()) { + if (isMultiQuery) { + rules.add("contains(., 'dvQuery=" + fieldName + ": [')"); + } else { + rules.add("starts-with(., 'SortedNumericDocValuesSetQuery(" + fieldName + ": [')"); + } + } + return String.join(" and ", rules); + } + + private void doTestDateFieldExactQuery(final String field, final String baseDate) + throws Exception { + doTestDateFieldExactQuery(field, baseDate, true); + } + + /** + * @param field the field to use for indexing and searching against + * @param baseDate basic value to use for indexing and searching + * @param searchable set to true if searches against "field" should succeed, false if field is + * only stored and searches should always get numFound=0 + */ + private void doTestDateFieldExactQuery( + final String field, final String baseDate, final boolean searchable) { + final String MATCH_ONE = "//*[@numFound='" + (searchable ? "1" : "0") + "']"; + final String MATCH_TWO = "//*[@numFound='" + (searchable ? "2" : "0") + "']"; + + for (int i = 0; i < 10; i++) { + assertU( + adoc( + "id", + String.valueOf(i), + field, + String.format(Locale.ROOT, "%s+%dMINUTES", baseDate, i + 1))); + } + assertU(commit()); + for (int i = 0; i < 10; i++) { + String date = String.format(Locale.ROOT, "%s+%dMINUTES", baseDate, i + 1); + assertQ(req("q", field + ":\"" + date + "\"", "fl", "id, " + field), MATCH_ONE); + } + + for (int i = 0; i < 10; i++) { + String date1 = String.format(Locale.ROOT, "%s+%dMINUTES", baseDate, i + 1); + String date2 = String.format(Locale.ROOT, "%s+%dMINUTES", baseDate, ((i + 1) % 10 + 1)); + assertQ( + req("q", field + ":\"" + date1 + "\"" + " OR " + field + ":\"" + date2 + "\""), + MATCH_TWO); + } + + clearIndex(); + assertU(commit()); + } + + private void doTestDateFieldRangeQuery(String fieldName) { + String baseDate = "1995-12-31T10:59:59Z"; + for (int i = 9; i >= 0; i--) { + assertU( + adoc( + "id", + String.valueOf(i), + fieldName, + String.format(Locale.ROOT, "%s+%dHOURS", baseDate, i))); + } + assertU(commit()); + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+0HOURS TO %s+3HOURS]", baseDate, baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']", + "//result/doc[2]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']", + "//result/doc[3]/date[@name='" + fieldName + "'][.='1995-12-31T12:59:59Z']", + "//result/doc[4]/date[@name='" + fieldName + "'][.='1995-12-31T13:59:59Z']"); + + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "{%s+0HOURS TO %s+3HOURS]", baseDate, baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']", + "//result/doc[2]/date[@name='" + fieldName + "'][.='1995-12-31T12:59:59Z']", + "//result/doc[3]/date[@name='" + fieldName + "'][.='1995-12-31T13:59:59Z']"); + + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+0HOURS TO %s+3HOURS}", baseDate, baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']", + "//result/doc[2]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']", + "//result/doc[3]/date[@name='" + fieldName + "'][.='1995-12-31T12:59:59Z']"); + + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "{%s+0HOURS TO %s+3HOURS}", baseDate, baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='2']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']", + "//result/doc[2]/date[@name='" + fieldName + "'][.='1995-12-31T12:59:59Z']"); + + assertQ( + req( + "q", + fieldName + ":" + String.format(Locale.ROOT, "{%s+0HOURS TO *}", baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='9']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']"); + + assertQ( + req( + "q", + fieldName + ":" + String.format(Locale.ROOT, "{* TO %s+3HOURS}", baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']"); + + assertQ( + req( + "q", + fieldName + ":" + String.format(Locale.ROOT, "[* TO %s+3HOURS}", baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='3']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']"); + + assertQ( + req("q", fieldName + ":[* TO *}", "fl", "id, " + fieldName, "sort", "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']", + "//result/doc[10]/date[@name='" + fieldName + "'][.='1995-12-31T19:59:59Z']"); + + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+0HOURS TO %s+1HOURS]", baseDate, baseDate) + + " OR " + + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+8HOURS TO %s+9HOURS]", baseDate, baseDate), + "fl", + "id, " + fieldName, + "sort", + "id asc"), + "//*[@numFound='4']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']", + "//result/doc[2]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']", + "//result/doc[3]/date[@name='" + fieldName + "'][.='1995-12-31T18:59:59Z']", + "//result/doc[4]/date[@name='" + fieldName + "'][.='1995-12-31T19:59:59Z']"); + + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+0HOURS TO %s+1HOURS]", baseDate, baseDate) + + " AND " + + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+1HOURS TO %s+2HOURS]", baseDate, baseDate), + "fl", + "id, " + fieldName), + "//*[@numFound='1']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T11:59:59Z']"); + + assertQ( + req( + "q", + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+0HOURS TO %s+1HOURS]", baseDate, baseDate) + + " AND NOT " + + fieldName + + ":" + + String.format(Locale.ROOT, "[%s+1HOURS TO %s+2HOURS]", baseDate, baseDate), + "fl", + "id, " + fieldName), + "//*[@numFound='1']", + "//result/doc[1]/date[@name='" + fieldName + "'][.='1995-12-31T10:59:59Z']"); + + clearIndex(); + assertU(commit()); + + String[] arr = toAscendingStringArray(getRandomInstants(100, false), true); + for (int i = 0; i < arr.length; ++i) { + assertU(adoc("id", String.valueOf(i), fieldName, arr[i])); + } + assertU(commit()); + for (int i = 0; i < arr.length; ++i) { + assertQ( + req("q", fieldName + ":[" + arr[0] + " TO " + arr[i] + "]", "fl", "id," + fieldName), + "//*[@numFound='" + (i + 1) + "']"); + assertQ( + req("q", fieldName + ":{" + arr[0] + " TO " + arr[i] + "}", "fl", "id, " + fieldName), + "//*[@numFound='" + (Math.max(0, i - 1)) + "']"); + assertQ( + req( + "q", + fieldName + ":[" + arr[0] + " TO " + arr[i] + "] AND " + fieldName + ":\"" + arr[0] + + "\"", + "fl", + "id, " + fieldName), + "//*[@numFound='1']"); + } + } + + private void doTestDateFieldFunctionQuery(String field) { + // This method is intentionally not randomized, because sorting by function happens + // at float precision, which causes ms(date) to give the same value for different dates. + // See https://issues.apache.org/jira/browse/SOLR-11825 + + final String baseDate = "1995-01-10T10:59:10Z"; + + for (int i = 9; i >= 0; i--) { + String date = String.format(Locale.ROOT, "%s+%dSECONDS", baseDate, i + 1); + assertU(adoc("id", String.valueOf(i), field, date)); + } + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field, + "sort", + "product(-1,ms(" + field + "," + baseDate + ")) asc"), + "//*[@numFound='10']", + "//result/doc[1]/date[@name='" + field + "'][.='1995-01-10T10:59:20Z']", + "//result/doc[2]/date[@name='" + field + "'][.='1995-01-10T10:59:19Z']", + "//result/doc[3]/date[@name='" + field + "'][.='1995-01-10T10:59:18Z']", + "//result/doc[10]/date[@name='" + field + "'][.='1995-01-10T10:59:11Z']"); + + assertQ( + req( + "q", + "*:*", + "fl", + "id, " + field + ", ms(" + field + "," + baseDate + ")", + "sort", + "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/float[@name='ms(" + field + "," + baseDate + ")'][.='1000.0']", + "//result/doc[2]/float[@name='ms(" + field + "," + baseDate + ")'][.='2000.0']", + "//result/doc[3]/float[@name='ms(" + field + "," + baseDate + ")'][.='3000.0']", + "//result/doc[10]/float[@name='ms(" + field + "," + baseDate + ")'][.='10000.0']"); + + assertQ( + req("q", "*:*", "fl", "id, " + field + ", field(" + field + ")", "sort", "id asc"), + "//*[@numFound='10']", + "//result/doc[1]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:11Z']", + "//result/doc[2]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:12Z']", + "//result/doc[3]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:13Z']", + "//result/doc[10]/date[@name='field(" + field + ")'][.='1995-01-10T10:59:20Z']"); + } + + private void doTestDateFieldStats(String field, String dvField, String[] dates) { + for (int i = 0; i < dates.length; i++) { + assertU(adoc("id", String.valueOf(i), dvField, dates[i], field, dates[i])); + } + assertU(adoc("id", String.valueOf(dates.length))); + assertU(commit()); + assertTrue(h.getCore().getLatestSchema().getField(dvField).hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField(dvField).getType() instanceof NumericField); + assertQ( + req("q", "*:*", "fl", "id, " + dvField, "stats", "true", "stats.field", dvField), + "//*[@numFound='" + (dates.length + 1) + "']", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/date[@name='min'][.='" + + dates[0] + + "']", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/date[@name='max'][.='" + + dates[dates.length - 1] + + "']", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/long[@name='count'][.='" + + dates.length + + "']", + "//lst[@name='stats']/lst[@name='stats_fields']/lst[@name='" + + dvField + + "']/long[@name='missing'][.='1']"); + + assertFalse(h.getCore().getLatestSchema().getField(field).hasDocValues()); + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + assertQEx( + "Expecting Exception", + "Can't calculate stats on a NumericField without docValues", + req("q", "*:*", "fl", "id, " + field, "stats", "true", "stats.field", field), + SolrException.ErrorCode.BAD_REQUEST); + } + + private void doTestDateFieldsAtomicUpdates(String field) { + long millis1 = random().nextLong() % MAX_DATE_EPOCH_MILLIS; + long millis2; + DateGapCeiling gap; + for (; ; ) { + millis2 = random().nextLong() % MAX_DATE_EPOCH_MILLIS; + gap = new DateGapCeiling(millis2 - millis1); + millis2 = gap.addTo(millis1); // adjust millis2 to the closest +/-UNIT gap + break; + } + String date1 = Instant.ofEpochMilli(millis1).toString(); + String date2 = Instant.ofEpochMilli(millis2).toString(); + assertU(adoc(sdoc("id", "1", field, date1))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/date[@name='" + field + "'][.='" + date1 + "']"); + + assertU(adoc(sdoc("id", "1", field, Map.of("set", date1 + gap)))); + assertU(commit()); + + assertQ(req("q", "id:1"), "//result/doc[1]/date[@name='" + field + "'][.='" + date2 + "']"); + } + + private void doTestInternals(String field, String[] values) throws IOException { + assertTrue(h.getCore().getLatestSchema().getField(field).getType() instanceof NumericField); + for (int i = 0; i < 10; i++) { + assertU(adoc("id", String.valueOf(i), field, values[i])); + } + assertU(commit()); + + SchemaField sf = h.getCore().getLatestSchema().getField(field); + boolean ignoredField = !(sf.indexed() || sf.stored() || sf.hasDocValues()); + h.getCore() + .withSearcher( + searcher -> { + DirectoryReader ir = searcher.getIndexReader(); + // our own SlowCompositeReader to check DocValues on disk w/o the UninvertingReader + // added by SolrIndexSearcher + final LeafReader leafReaderForCheckingDVs = + SlowCompositeReaderWrapper.wrap(searcher.getRawReader()); + + if (sf.indexed()) { + assertEquals( + "Field " + field + " should have point values", + 10, + PointValues.size(ir, field)); + } else { + assertEquals( + "Field " + field + " should have no point values", + 0, + PointValues.size(ir, field)); + } + if (ignoredField) { + assertEquals( + "Field " + field + " should not have docValues", + DocIdSetIterator.NO_MORE_DOCS, + DocValues.getSortedNumeric(leafReaderForCheckingDVs, field).nextDoc()); + assertEquals( + "Field " + field + " should not have docValues", + DocIdSetIterator.NO_MORE_DOCS, + DocValues.getNumeric(leafReaderForCheckingDVs, field).nextDoc()); + assertEquals( + "Field " + field + " should not have docValues", + DocIdSetIterator.NO_MORE_DOCS, + DocValues.getSorted(leafReaderForCheckingDVs, field).nextDoc()); + assertEquals( + "Field " + field + " should not have docValues", + DocIdSetIterator.NO_MORE_DOCS, + DocValues.getBinary(leafReaderForCheckingDVs, field).nextDoc()); + } else { + if (sf.hasDocValues()) { + if (sf.multiValued()) { + assertNotEquals( + "Field " + field + " should have docValues", + DocIdSetIterator.NO_MORE_DOCS, + DocValues.getSortedNumeric(leafReaderForCheckingDVs, field).nextDoc()); + } else { + assertNotEquals( + "Field " + field + " should have docValues", + DocIdSetIterator.NO_MORE_DOCS, + DocValues.getNumeric(leafReaderForCheckingDVs, field).nextDoc()); + } + } else { + expectThrows( + IllegalStateException.class, + () -> DocValues.getSortedNumeric(leafReaderForCheckingDVs, field)); + expectThrows( + IllegalStateException.class, + () -> DocValues.getNumeric(leafReaderForCheckingDVs, field)); + } + expectThrows( + IllegalStateException.class, + () -> DocValues.getSorted(leafReaderForCheckingDVs, field)); + expectThrows( + IllegalStateException.class, + () -> DocValues.getBinary(leafReaderForCheckingDVs, field)); + } + for (LeafReaderContext leave : ir.leaves()) { + LeafReader reader = leave.reader(); + StoredFields storedFields = reader.storedFields(); + for (int i = 0; i < reader.numDocs(); i++) { + Document doc = storedFields.document(i); + if (sf.stored()) { + assertNotNull("Field " + field + " not found. Doc: " + doc, doc.get(field)); + } else { + assertNull(doc.get(field)); + } + } + } + return null; + }); + clearIndex(); + assertU(commit()); + } + + public void testNonReturnable() throws Exception { + String[] ints = toStringArray(getRandomInts(2, false)); + doTestReturnNonStored("foo_p_i_ni_ns", false, ints[0]); + doTestReturnNonStored("foo_p_i_ni_dv_ns", true, ints[0]); + doTestReturnNonStored("foo_p_i_ni_ns_mv", false, ints); + doTestReturnNonStored("foo_p_i_ni_dv_ns_mv", true, ints); + + String[] longs = toStringArray(getRandomLongs(2, false)); + doTestReturnNonStored("foo_p_l_ni_ns", false, longs[0]); + doTestReturnNonStored("foo_p_l_ni_dv_ns", true, longs[0]); + doTestReturnNonStored("foo_p_l_ni_ns_mv", false, longs); + doTestReturnNonStored("foo_p_l_ni_dv_ns_mv", true, longs); + + String[] floats = toStringArray(getRandomFloats(2, false)); + doTestReturnNonStored("foo_p_f_ni_ns", false, floats[0]); + doTestReturnNonStored("foo_p_f_ni_dv_ns", true, floats[0]); + doTestReturnNonStored("foo_p_f_ni_ns_mv", false, floats); + doTestReturnNonStored("foo_p_f_ni_dv_ns_mv", true, floats); + + String[] doubles = toStringArray(getRandomDoubles(2, false)); + doTestReturnNonStored("foo_p_d_ni_ns", false, doubles[0]); + doTestReturnNonStored("foo_p_d_ni_dv_ns", true, doubles[0]); + doTestReturnNonStored("foo_p_d_ni_ns_mv", false, doubles); + doTestReturnNonStored("foo_p_d_ni_dv_ns_mv", true, doubles); + + String[] dates = new String[] {getRandomDateMaybeWithMath(), getRandomDateMaybeWithMath()}; + doTestReturnNonStored("foo_p_dt_ni_ns", false, dates[0]); + doTestReturnNonStored("foo_p_dt_ni_dv_ns", true, dates[0]); + doTestReturnNonStored("foo_p_dt_ni_ns_mv", false, dates); + doTestReturnNonStored("foo_p_dt_ni_dv_ns_mv", true, dates); + } + + public void doTestReturnNonStored( + final String fieldName, boolean shouldReturnFieldIfRequested, final String... values) { + final String RETURN_FIELD = "count(//doc/*[@name='" + fieldName + "'])=10"; + final String DONT_RETURN_FIELD = "count(//doc/*[@name='" + fieldName + "'])=0"; + assertFalse(h.getCore().getLatestSchema().getField(fieldName).stored()); + for (int i = 0; i < 10; i++) { + SolrInputDocument doc = sdoc("id", String.valueOf(i)); + for (String value : values) { + doc.addField(fieldName, value); + } + assertU(adoc(doc)); + } + assertU(commit()); + assertQ( + req("q", "*:*", "rows", "100", "fl", "id," + fieldName), + "//*[@numFound='10']", + "count(//doc)=10", // exactly 10 docs in response + (shouldReturnFieldIfRequested + ? RETURN_FIELD + : DONT_RETURN_FIELD)); // no field in any doc other than 'id' + + assertQ( + req("q", "*:*", "rows", "100", "fl", "*"), + "//*[@numFound='10']", + "count(//doc)=10", // exactly 10 docs in response + DONT_RETURN_FIELD); // no field in any doc other than 'id' + + assertQ( + req("q", "*:*", "rows", "100"), + "//*[@numFound='10']", + "count(//doc)=10", // exactly 10 docs in response + DONT_RETURN_FIELD); // no field in any doc other than 'id' + clearIndex(); + assertU(commit()); + } + + public void testWhiteboxCreateFields() throws Exception { + String[] typeNames = new String[] {"i", "l", "f", "d", "dt"}; + Class[] expectedClasses = + new Class[] { + IntPoint.class, LongPoint.class, FloatPoint.class, DoublePoint.class, LongPoint.class + }; + + Date dateToTest = new Date(); + Object[][] values = + new Object[][] { + {42, "42"}, + {42, "42"}, + {42.123, "42.123"}, + {12345.6789, "12345.6789"}, + { + dateToTest, + new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.ROOT).format(dateToTest), + "NOW" + } // "NOW" won't be equal to the other dates + }; + + Set typesTested = new HashSet<>(); + for (int i = 0; i < typeNames.length; i++) { + for (String suffix : FIELD_SUFFIXES) { + doWhiteboxCreateFields( + "whitebox_p_" + typeNames[i] + suffix, expectedClasses[i], values[i]); + typesTested.add("*_p_" + typeNames[i] + suffix); + } + } + Set typesToTest = new HashSet<>(); + for (DynamicField dynField : h.getCore().getLatestSchema().getDynamicFields()) { + if (dynField.getPrototype().getType() instanceof NumericField) { + typesToTest.add(dynField.getRegex()); + } + } + assertEquals("Missing types in the test", typesTested, typesToTest); + } + + /** + * Calls {@link #callAndCheckCreateFields} on each of the specified values. This is a convenience + * method for testing the same fieldname with multiple inputs. + * + * @see #callAndCheckCreateFields + */ + private void doWhiteboxCreateFields( + final String fieldName, final Class pointType, final Object... values) throws Exception { + + for (Object value : values) { + // ideally we should require that all input values be diff forms of the same logical value (ie + // '"42"' vs 'new Integer(42)') and assert that each produces an equivalent list of + // IndexableField objects but that doesn't seem to work -- appears not all IndexableField + // classes override Object.equals? + final List result = callAndCheckCreateFields(fieldName, pointType, value); + assertNotNull(value + " => null", result); + } + } + + /** + * Calls {@link SchemaField#createFields} on the specified value for the specified field name, and + * asserts that the results match the SchemaField properties, with an additional check that the + * pointType is included if and only if the SchemaField is "indexed" + */ + private List callAndCheckCreateFields( + final String fieldName, final Class pointType, final Object value) { + final SchemaField sf = h.getCore().getLatestSchema().getField(fieldName); + final List results = sf.createFields(value); + final Set resultSet = new LinkedHashSet<>(results); + assertEquals("duplicates found in results? " + results, results.size(), resultSet.size()); + + final Set> resultClasses = new HashSet<>(); + for (IndexableField f : results) { + resultClasses.add(f.getClass()); + + if (!sf.hasDocValues()) { + assertFalse( + f.toString(), + (f instanceof NumericDocValuesField) || (f instanceof SortedNumericDocValuesField)); + } + } + assertEquals( + fieldName + " stored? Result Fields: " + Arrays.toString(results.toArray()), + sf.stored(), + resultClasses.contains(StoredField.class)); + assertEquals( + fieldName + " indexed? Result Fields: " + Arrays.toString(results.toArray()), + sf.indexed(), + resultClasses.contains(pointType)); + if (sf.multiValued()) { + assertEquals( + fieldName + " docvalues? Result Fields: " + Arrays.toString(results.toArray()), + sf.hasDocValues(), + resultClasses.contains(SortedNumericDocValuesField.class)); + } else { + assertEquals( + fieldName + " docvalues? Result Fields: " + Arrays.toString(results.toArray()), + sf.hasDocValues(), + resultClasses.contains(NumericDocValuesField.class)); + } + + return results; + } +} diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SecurityNodeWatcher.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SecurityNodeWatcher.java index 487fa6b427b..9edfbc6e249 100644 --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SecurityNodeWatcher.java +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/SecurityNodeWatcher.java @@ -21,7 +21,7 @@ import java.lang.invoke.MethodHandles; import java.nio.charset.StandardCharsets; import java.util.Map; -import org.apache.solr.common.Callable; +import java.util.function.Consumer; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.Utils; import org.apache.zookeeper.KeeperException; @@ -37,7 +37,7 @@ class SecurityNodeWatcher implements Watcher { private final ZkStateReader zkStateReader; private ZkStateReader.ConfigData securityData; - private final Callable callback; + private final Consumer callback; @SuppressWarnings("unchecked") public SecurityNodeWatcher(ZkStateReader zkStateReader, Runnable securityNodeListener) { @@ -79,7 +79,7 @@ public void process(WatchedEvent event) { .getNode(ZkStateReader.SOLR_SECURITY_CONF_PATH, this, true); } try { - callback.call(data); + callback.accept(data); } catch (Exception e) { log.error("Error running collections node listener", e); } diff --git a/solr/solrj/src/java/org/apache/solr/common/Callable.java b/solr/solrj/src/java/org/apache/solr/common/Callable.java deleted file mode 100644 index 8e43ac8e391..00000000000 --- a/solr/solrj/src/java/org/apache/solr/common/Callable.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.common; - -public interface Callable { - public void call(T data); // data depends on the context -}