Stem prefix items

config-model/src/main/java/com/yahoo/schema/document/Matching.java

@@ -31,6 +31,7 @@ public class Matching implements Cloneable, Serializable {
 
  /** Maximum number of characters to consider when searching in this field. Used for limiting resources, especially in streaming search. */
  private Integer maxLength;
+
  /** Maximum number of occurrences for each term */
  private Integer maxTermOccurrences;
 

config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java

@@ -23,8 +23,7 @@ public class NoPrefixForIndexes implements Validator {
  @Override
  public void validate(Context context) {
  for (SearchCluster cluster : context.model().getSearchClusters()) {
- if (cluster instanceof IndexedSearchCluster) {
- IndexedSearchCluster sc = (IndexedSearchCluster) cluster;
+ if (cluster instanceof IndexedSearchCluster sc) {
  for (DocumentDatabase docDb : sc.getDocumentDbs()) {
  DerivedConfiguration sdConfig = docDb.getDerivedConfiguration();
  Schema schema = sdConfig.getSchema();

container-search/abi-spec.json

@@ -524,9 +524,12 @@
  "methods" : [
  "public void <init>(java.lang.String)",
  "public void <init>(java.lang.String, boolean)",
+ "public void <init>(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
+ "public com.yahoo.prelude.query.ExactStringItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public com.yahoo.prelude.query.Item$ItemType getItemType()",
  "public java.lang.String getName()",
- "public java.lang.String stringValue()"
+ "public java.lang.String stringValue()",
+ "public bridge synthetic com.yahoo.prelude.query.WordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)"
  ],
  "fields" : [ ]
  },
@@ -914,6 +917,7 @@
  "public"
  ],
  "methods" : [
+ "public com.yahoo.prelude.query.MarkerWordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public boolean isStartAnchor()",
  "public boolean isEndAnchor()",
  "protected java.lang.String getEncodedWord()",
@@ -923,7 +927,8 @@
  "public static com.yahoo.prelude.query.MarkerWordItem createStartOfHost(java.lang.String)",
  "public static com.yahoo.prelude.query.MarkerWordItem createStartOfHost()",
  "public static com.yahoo.prelude.query.MarkerWordItem createEndOfHost(java.lang.String)",
- "public static com.yahoo.prelude.query.MarkerWordItem createEndOfHost()"
+ "public static com.yahoo.prelude.query.MarkerWordItem createEndOfHost()",
+ "public bridge synthetic com.yahoo.prelude.query.WordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)"
  ],
  "fields" : [ ]
  },
@@ -1296,9 +1301,12 @@
  "public void <init>(java.lang.String)",
  "public void <init>(java.lang.String, boolean)",
  "public void <init>(java.lang.String, java.lang.String)",
+ "public void <init>(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
+ "public com.yahoo.prelude.query.PrefixItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public com.yahoo.prelude.query.Item$ItemType getItemType()",
  "public java.lang.String getName()",
- "public java.lang.String stringValue()"
+ "public java.lang.String stringValue()",
+ "public bridge synthetic com.yahoo.prelude.query.WordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)"
  ],
  "fields" : [ ]
  },
@@ -1622,9 +1630,12 @@
  "methods" : [
  "public void <init>(java.lang.String)",
  "public void <init>(java.lang.String, boolean)",
+ "public void <init>(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
+ "public com.yahoo.prelude.query.SubstringItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public com.yahoo.prelude.query.Item$ItemType getItemType()",
  "public java.lang.String getName()",
- "public java.lang.String stringValue()"
+ "public java.lang.String stringValue()",
+ "public bridge synthetic com.yahoo.prelude.query.WordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)"
  ],
  "fields" : [ ]
  },
@@ -1637,9 +1648,12 @@
  "methods" : [
  "public void <init>(java.lang.String)",
  "public void <init>(java.lang.String, boolean)",
+ "public void <init>(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
+ "public com.yahoo.prelude.query.SuffixItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public com.yahoo.prelude.query.Item$ItemType getItemType()",
  "public java.lang.String getName()",
- "public java.lang.String stringValue()"
+ "public java.lang.String stringValue()",
+ "public bridge synthetic com.yahoo.prelude.query.WordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)"
  ],
  "fields" : [ ]
  },
@@ -1962,6 +1976,7 @@
  "public void <init>(com.yahoo.prelude.query.parser.Token, boolean)",
  "public void <init>(java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public void <init>(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
+ "public com.yahoo.prelude.query.WordItem newInstance(java.lang.String, java.lang.String, boolean, com.yahoo.prelude.query.Substring)",
  "public com.yahoo.prelude.query.Item$ItemType getItemType()",
  "public java.lang.String getName()",
  "public void setWord(java.lang.String)",

container-search/src/main/java/com/yahoo/prelude/query/ExactStringItem.java

@@ -17,6 +17,14 @@ public ExactStringItem(String substring, boolean isFromQuery) {
  super(substring, isFromQuery);
  }
 
+ public ExactStringItem(String word, String indexName, boolean isFromQuery, Substring origin) {
+ super(word, indexName, isFromQuery, origin);
+ }
+
+ public ExactStringItem newInstance(String word, String indexName, boolean isFromQuery, Substring origin) {
+ return new ExactStringItem(word, indexName, isFromQuery, origin);
+ }
+
  @Override
  public ItemType getItemType() {
  return ItemType.EXACT;

container-search/src/main/java/com/yahoo/prelude/query/MarkerWordItem.java

@@ -25,6 +25,16 @@ private MarkerWordItem(String publicSymbol, String markerWord, String indexName)
  this.markerWord = markerWord;
  }
 
+ private MarkerWordItem(String publicSymbol, String markerWord, String indexName, boolean isFromQuery, Substring origin) {
+ super(publicSymbol, indexName);
+ this.markerWord = markerWord;
+ }
+
+ /** Returns a new instance of this kind of WordItem, initialized with the given data and nothing else. */
+ public MarkerWordItem newInstance(String word, String indexName, boolean isFromQuery, Substring origin) {
+ return new MarkerWordItem(word, markerWord, indexName, isFromQuery, origin);
+ }
+
  public boolean isStartAnchor() { return getWord().equals(startAnchor); }
 
  public boolean isEndAnchor() { return getWord().equals(endAnchor); }

container-search/src/main/java/com/yahoo/prelude/query/PrefixItem.java

@@ -17,7 +17,17 @@ public PrefixItem(String prefix, boolean isFromQuery) {
  super(prefix, isFromQuery);
  }
 
- public PrefixItem(String prefix, String indexName) { super(prefix, indexName); }
+ public PrefixItem(String prefix, String indexName) {
+ super(prefix, indexName);
+ }
+
+ public PrefixItem(String prefix, String indexName, boolean isFromQuery, Substring origin) {
+ super(prefix, indexName, isFromQuery, origin);
+ }
+
+ public PrefixItem newInstance(String word, String indexName, boolean isFromQuery, Substring origin) {
+ return new PrefixItem(word, indexName, isFromQuery, origin);
+ }
 
  @Override
  public ItemType getItemType() {

container-search/src/main/java/com/yahoo/prelude/query/SubstringItem.java

@@ -16,6 +16,14 @@ public SubstringItem(String substring, boolean isFromQuery) {
  super(substring, isFromQuery);
  }
 
+ public SubstringItem(String substring, String indexName, boolean isFromQuery, Substring origin) {
+ super(substring, indexName, isFromQuery, origin);
+ }
+
+ public SubstringItem newInstance(String word, String indexName, boolean isFromQuery, Substring origin) {
+ return new SubstringItem(word, indexName, isFromQuery, origin);
+ }
+
  @Override
  public ItemType getItemType() {
  return ItemType.SUBSTRING;

container-search/src/main/java/com/yahoo/prelude/query/SuffixItem.java

@@ -16,6 +16,14 @@ public SuffixItem(String suffix, boolean isFromQuery) {
  super(suffix, isFromQuery);
  }
 
+ public SuffixItem(String substring, String indexName, boolean isFromQuery, Substring origin) {
+ super(substring, indexName, isFromQuery, origin);
+ }
+
+ public SuffixItem newInstance(String word, String indexName, boolean isFromQuery, Substring origin) {
+ return new SuffixItem(word, indexName, isFromQuery, origin);
+ }
+
  @Override
  public ItemType getItemType() {
  return ItemType.SUFFIX;

container-search/src/main/java/com/yahoo/prelude/query/WordItem.java

@@ -62,6 +62,14 @@ public WordItem(String word, String indexName, boolean isFromQuery, Substring or
  setWord(word);
  }
 
+ /**
+ * Returns a new instance of this kind of WordItem, initialized with the given data and any other
+ * fields belonging to the item subclass copied from this instance.
+ */
+ public WordItem newInstance(String word, String indexName, boolean isFromQuery, Substring origin) {
+ return new WordItem(word, indexName, isFromQuery, origin);
+ }
+
  public ItemType getItemType() {
  return ItemType.WORD;
  }

container-search/src/main/java/com/yahoo/prelude/querytransform/StemmingSearcher.java

@@ -163,7 +163,7 @@ private Item scan(Item item, StemContext context) {
  }
 
  private Item checkBlock(BlockItem b, StemContext context) {
- if (b instanceof PrefixItem || !b.isWords()) return (Item) b;
+ if (!b.isWords()) return (Item) b;
 
  if (b.isFromQuery() && !b.isStemmed()) {
  Index index = context.indexFacts.getIndex(b.getIndexName());
@@ -190,10 +190,8 @@ private Substring getOffsets(BlockItem b) {
 
  // The rewriting logic is here
  private Item stem(BlockItem current, StemContext context, Index index) {
- Item blockAsItem = (Item)current;
- CompositeItem composite;
  List<StemList> segments = linguistics.getStemmer().stem(current.stringValue(), index.getStemMode(), context.language);
- if (segments.isEmpty()) return blockAsItem;
+ if (segments.isEmpty()) return (Item)current;
 
  String indexName = current.getIndexName();
  Substring substring = getOffsets(current);
@@ -203,6 +201,7 @@ private Item stem(BlockItem current, StemContext context, Index index) {
  return (Item)w;
  }
 
+ CompositeItem composite;
  if (context.isCJK)
  composite = chooseCompositeForCJK(current, ((Item) current).getParent(), indexName);
  else
@@ -219,7 +218,7 @@ private Item stem(BlockItem current, StemContext context, Index index) {
  if (composite instanceof AndSegmentItem) {
  andSegmentConnectivity(current, context.reverseConnectivity, composite);
  }
- copyAttributes(blockAsItem, composite);
+ copyAttributes((Item)current, composite);
  composite.lock();
 
  if (composite instanceof PhraseSegmentItem replacement) {
@@ -320,7 +319,11 @@ private void setMetaData(BlockItem current, Map<Item, TaggableItem> reverseConne
 
  private WordItem singleStemSegment(Item blockAsItem, String stem, String indexName,
  Substring substring) {
- WordItem replacement = new WordItem(stem, indexName, true, substring);
+ WordItem replacement;
+ if (blockAsItem instanceof WordItem) // preserve the WordItem subclass type
+ replacement = ((WordItem)blockAsItem).newInstance(stem, indexName, true, substring);
+ else
+ replacement = new WordItem(stem, indexName, true, substring);
  replacement.setStemmed(true);
  copyAttributes(blockAsItem, replacement);
  return replacement;