Skip to content

SOLR-12074: Add optional terms index for PointFields #3922

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HoustonPutman wants to merge 5 commits into
base: main
Choose a base branch
from
Open

SOLR-12074: Add optional terms index for PointFields #3922

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
0ab6359
SOLR-12074: Add optional terms index for PointFields
HoustonPutman Dec 4, 2025
6a72fad
Tidy
HoustonPutman Dec 4, 2025
e188171
Add tries to benchmark
HoustonPutman Dec 4, 2025
703d497
Add testing in TestPointFields
HoustonPutman Dec 4, 2025
77798d1
Tidy
HoustonPutman Dec 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions changelog/unreleased/solr-12074-point-field-terms.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
title: Add option to index Point field terms, to speed up term lookup and close funcitonality gap with Trie fields
type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other
authors:
- name: Houston Putman
nick: HoustonPutman
url: https://home.apache.org/phonebook.html?uid=houston
links:
- name: SOLR-12074
url: https://issues.apache.org/jira/browse/SOLR-12074
156 changes: 140 additions & 16 deletions solr/benchmark/src/java/org/apache/solr/bench/search/NumericSearch.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,20 +147,24 @@ public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterSta
miniClusterState.client.requestWithBaseUrl(miniClusterState.nodes.get(0), reload, null);
}

public QueryRequest intSetQuery(boolean dvs) {
return setQuery("numbers_i" + (dvs ? "_dv" : ""));
public QueryRequest intTrieSetQuery(boolean dvs, boolean enhancedIndex) {
Copy link
Contributor

@gerlowskija gerlowskija Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Q] Why provide the "enhancedIndex" flag here and then not use it?

return setQuery("numbers_it" + (dvs ? "_dv" : ""));
}

public QueryRequest longSetQuery(boolean dvs) {
return setQuery("numbers_l" + (dvs ? "_dv" : ""));
public QueryRequest intSetQuery(boolean dvs, boolean enhancedIndex) {
return setQuery("numbers_i" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : ""));
}

public QueryRequest doubleSetQuery(boolean dvs) {
return setQuery("numbers_d" + (dvs ? "_dv" : ""));
public QueryRequest longSetQuery(boolean dvs, boolean enhancedIndex) {
return setQuery("numbers_l" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : ""));
}

public QueryRequest floatSetQuery(boolean dvs) {
return setQuery("numbers_f" + (dvs ? "_dv" : ""));
public QueryRequest doubleSetQuery(boolean dvs, boolean enhancedIndex) {
return setQuery("numbers_d" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : ""));
}

public QueryRequest floatSetQuery(boolean dvs, boolean enhancedIndex) {
return setQuery("numbers_f" + (dvs ? "_dv" : "") + (enhancedIndex ? "_e" : ""));
}

QueryRequest setQuery(String field) {
Expand All @@ -175,14 +179,38 @@ QueryRequest setQuery(String field) {
}
}

@Benchmark
public Object intTrieSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.intTrieSetQuery(false, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object intTrieDvSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.intTrieSetQuery(false, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object intSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.intSetQuery(false).process(miniClusterState.client, COLLECTION);
benchState.intSetQuery(false, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -194,7 +222,7 @@ public Object longSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.longSetQuery(false).process(miniClusterState.client, COLLECTION);
benchState.longSetQuery(false, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -206,7 +234,7 @@ public Object floatSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.floatSetQuery(false).process(miniClusterState.client, COLLECTION);
benchState.floatSetQuery(false, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -218,7 +246,7 @@ public Object doubleSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.doubleSetQuery(false).process(miniClusterState.client, COLLECTION);
benchState.doubleSetQuery(false, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -230,7 +258,7 @@ public Object intDvSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.intSetQuery(true).process(miniClusterState.client, COLLECTION);
benchState.intSetQuery(true, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -242,7 +270,7 @@ public Object longDvSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.longSetQuery(true).process(miniClusterState.client, COLLECTION);
benchState.longSetQuery(true, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -254,7 +282,7 @@ public Object floatDvSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.floatSetQuery(true).process(miniClusterState.client, COLLECTION);
benchState.floatSetQuery(true, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand All @@ -266,7 +294,103 @@ public Object doubleDvSet(
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.doubleSetQuery(true).process(miniClusterState.client, COLLECTION);
benchState.doubleSetQuery(true, false).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object intEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.intSetQuery(false, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object longEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.longSetQuery(false, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object floatEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.floatSetQuery(false, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object doubleEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.doubleSetQuery(false, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object intDvEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.intSetQuery(true, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object longDvEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.longSetQuery(true, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object floatDvEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.floatSetQuery(true, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}

@Benchmark
public Object doubleDvEnhancedSet(
Blackhole blackhole,
BenchState benchState,
MiniClusterState.MiniClusterBenchState miniClusterState)
throws SolrServerException, IOException {
QueryResponse response =
benchState.doubleSetQuery(true, true).process(miniClusterState.client, COLLECTION);
blackhole.consume(response);
return response;
}
Expand Down
18 changes: 16 additions & 2 deletions solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,13 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
<schema name="minimal" version="1.7">
<schema name="minimal" version="1.6">
Copy link
Contributor

@gerlowskija gerlowskija Dec 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Q] Why is this going backwards from 1.7?

Copy link
Contributor Author

@HoustonPutman HoustonPutman Dec 8, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ahhh yeah, because it was testing docValues or not, but the docValues are enabled by docValues=true which is default in 1.7. So instead of going and changing to docValues=false, I just downgraded. I'll make it better before merging for sure. Same with the enhancedIndex flag above.

<fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
<fieldType name="string" class="solr.StrField"/>
<fieldType name="int" class="org.apache.solr.schema.IntPointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
positionIncrementGap="0"/>
<fieldType name="inttrie" class="org.apache.solr.schema.TrieIntField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
<fieldType name="long" class="org.apache.solr.schema.LongPointField" docValues="false" omitNorms="true"
positionIncrementGap="0"/>
<fieldType name="float" class="org.apache.solr.schema.FloatPointField" docValues="false" omitNorms="true"
Expand All @@ -45,17 +47,29 @@
<dynamicField name="*_s" type="string" indexed="true" stored="false"/>
<dynamicField name="*_t" type="text" indexed="true" stored="false"/>
<dynamicField name="*_ts" type="text" indexed="true" stored="true"/>
<dynamicField name="*_it" type="inttrie" indexed="true" stored="false"/>
<dynamicField name="*_it_dv" type="inttrie" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_i" type="int" indexed="true" stored="false"/>
<dynamicField name="*_i_dv" type="int" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_i_e" type="int" indexed="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_i_dv_e" type="int" indexed="true" docValues="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_is" type="int" indexed="false" stored="true"/>
<dynamicField name="*_l" type="long" indexed="true" stored="false"/>
<dynamicField name="*_l_dv" type="long" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_l_e" type="long" indexed="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_l_dv_e" type="long" indexed="true" docValues="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_f" type="float" indexed="true" stored="false"/>
<dynamicField name="*_f_dv" type="float" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_f_e" type="float" indexed="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_f_dv_e" type="float" indexed="true" docValues="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_d" type="double" indexed="true" stored="false"/>
<dynamicField name="*_d_dv" type="double" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_d_e" type="double" indexed="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_d_dv_e" type="double" indexed="true" docValues="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_dt" type="date" indexed="true" stored="false"/>
<dynamicField name="*_dt_dv" type="date" indexed="true" docValues="true" stored="false"/>
<dynamicField name="*_dt_e" type="date" indexed="true" stored="false" enhancedIndex="true"/>
<dynamicField name="*_dt_dv_e" type="date" indexed="true" docValues="true" stored="false" enhancedIndex="true"/>

<uniqueKey>id</uniqueKey>
</schema>
48 changes: 31 additions & 17 deletions solr/core/src/java/org/apache/solr/schema/DatePointField.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import java.util.Collection;
import java.util.Date;
import java.util.Map;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.SortedNumericDocValuesField;
import org.apache.lucene.document.StoredField;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LeafReaderContext;
Expand All @@ -33,6 +33,7 @@
import org.apache.lucene.queries.function.docvalues.LongDocValues;
import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.queries.function.valuesource.MultiValuedLongFieldSource;
import org.apache.lucene.search.IndexOrDocValuesQuery;
import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.SortedNumericSelector;
Expand Down Expand Up @@ -157,28 +158,41 @@ public Object toObject(IndexableField f) {
}
}

@Override
protected Query getExactQuery(SchemaField field, String externalVal) {
return LongPoint.newExactQuery(
field.getName(), DateMathParser.parseMath(null, externalVal).getTime());
}

@Override
public Query getSetQuery(QParser parser, SchemaField field, Collection<String> externalVals) {
assert externalVals.size() > 0;
if (!field.indexed()) {
return super.getSetQuery(parser, field, externalVals);
}
long[] values = new long[externalVals.size()];
int i = 0;
for (String val : externalVals) {
values[i] = DateMathParser.parseMath(null, val).getTime();
i++;
Query indexQuery = null;
long[] values = null;
if (hasIndexedTerms(field)) {
indexQuery = super.getSetQuery(parser, field, externalVals);
} else if (field.indexed()) {
values = new long[externalVals.size()];
int i = 0;
for (String val : externalVals) {
values[i++] = DateMathParser.parseMath(null, val).getTime();
}
indexQuery = LongPoint.newSetQuery(field.getName(), values);
}
if (field.hasDocValues()) {
return LongField.newSetQuery(field.getName(), values);
long[] points = new long[externalVals.size()];
if (values != null) {
points = values.clone();
} else {
int i = 0;
for (String val : externalVals) {
points[i++] = DateMathParser.parseMath(null, val).getTime();
}
}
Query docValuesQuery = SortedNumericDocValuesField.newSlowSetQuery(field.getName(), points);
if (indexQuery != null) {
return new IndexOrDocValuesQuery(indexQuery, docValuesQuery);
} else {
return docValuesQuery;
}
} else if (indexQuery != null) {
return indexQuery;
} else {
return LongPoint.newSetQuery(field.getName(), values);
return super.getSetQuery(parser, field, externalVals);
}
}

Expand Down
Loading
Loading