Create runtime fields

clientlib/src/main/proto/yelp/nrtsearch/luceneserver.proto

@@ -516,6 +516,8 @@ enum FieldType {
     // List of float values
     VECTOR = 15;
     CONTEXT_SUGGEST = 16; //Field used for contextual suggest fields
+    // Runtime fields
+    RUNTIME = 17;
 }
 
 //How the tokens should be indexed.
@@ -1132,4 +1134,4 @@ message CustomRequest {
 
 message CustomResponse {
     map<string, string> response = 1; // Custom response sent by the plugin
-}
+}

clientlib/src/main/proto/yelp/nrtsearch/search.proto

@@ -452,6 +452,8 @@ message SearchRequest {
     bool explain = 25;
     // Search nested object fields for each hit
     map<string, InnerHit> inner_hits = 26;
+    // Defines runtime fields for this query
+    repeated RuntimeField runtimeFields = 27;
 }
 
 /* Inner Hit search request */
@@ -478,6 +480,14 @@ message VirtualField {
     string name = 2; // Virtual field's name. Must be different from registered fields and any other virtual fields.
 }
 
+/* Runtime field used during search */
+message RuntimeField {
+    // Script defining this field's values.
+    Script script = 1;
+    // Runtime field's name. Must be different from registered fields and any other runtime fields.
+    string name = 2;
+}
+
 message Script {
     string lang = 1; // script language
     string source = 2; // script source
@@ -598,12 +608,30 @@ message SearchResponse {
                 google.protobuf.Struct structValue = 8; // Value for structured data
                 // Value for VECTOR FieldType
                 Vector vectorValue = 9;
+                RepeatedFieldValues repeatedFieldValues = 10;
             }
 
             message Vector {
                 repeated float value = 1;
             }
 
+            message RepeatedFieldValues {
+                // Repeated value for ATOM and TEXT  FieldType
+                repeated string textValues = 1; 
+                // Repeated value for BOOLEAN FieldType
+                repeated bool booleanValues = 2;
+                // Repeated value for INT FieldType2; 
+                repeated int32 intValues = 3; 
+                // Repeated value for LONG and DATE_TIME (as milliseconds since epoch in UTC zone) FieldType
+                repeated int64 longValues = 4; 
+                // Repeated value for FLOAT FieldType
+                repeated float floatValues = 5; 
+                // Repeated value for DOUBLE FieldType
+                repeated double doubleValues = 6; 
+                // Repeated value for structured data
+                repeated google.protobuf.Struct structValues = 7; 
+            }
+
         }
 
         message CompositeFieldValue {

src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java

@@ -15,8 +15,11 @@
  */
 package com.yelp.nrtsearch.server.luceneserver;
 
+import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.protobuf.Struct;
+import com.google.protobuf.Struct.Builder;
+import com.google.protobuf.Value;
 import com.google.protobuf.util.JsonFormat;
 import com.yelp.nrtsearch.server.grpc.DeadlineUtils;
 import com.yelp.nrtsearch.server.grpc.FacetResult;
@@ -37,9 +40,11 @@
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.ObjectFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.PolygonfieldDef;
+import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.innerhit.InnerHitFetchTask;
 import com.yelp.nrtsearch.server.luceneserver.rescore.RescoreTask;
+import com.yelp.nrtsearch.server.luceneserver.script.RuntimeScript;
 import com.yelp.nrtsearch.server.luceneserver.search.FieldFetchContext;
 import com.yelp.nrtsearch.server.luceneserver.search.SearchContext;
 import com.yelp.nrtsearch.server.luceneserver.search.SearchCutoffWrapper.CollectionTimeoutException;
@@ -763,7 +768,6 @@ private CompositeFieldValue getFieldForHit(
 
       // We detect invalid field above:
       assert fd != null;
-
       if (fd instanceof VirtualFieldDef) {
         VirtualFieldDef virtualFieldDef = (VirtualFieldDef) fd;
 
@@ -794,6 +798,7 @@ public boolean advanceExact(int doc) throws IOException {
         for (int i = 0; i < docValues.size(); ++i) {
           compositeFieldValue.addFieldValue(docValues.toFieldValue(i));
         }
+
       }
       // retrieve stored fields
       else if (fd instanceof IndexableFieldDef && ((IndexableFieldDef) fd).isStored()) {
@@ -896,6 +901,12 @@ private static void fetchSlice(
               sliceSegment,
               fieldDefEntry.getKey(),
               (VirtualFieldDef) fieldDefEntry.getValue());
+        } else if (fieldDefEntry.getValue() instanceof RuntimeFieldDef) {
+          fetchRuntimeFromSegmentFactory(
+              sliceHits,
+              sliceSegment,
+              fieldDefEntry.getKey(),
+              (RuntimeFieldDef) fieldDefEntry.getValue());
         } else if (fieldDefEntry.getValue() instanceof IndexableFieldDef) {
           IndexableFieldDef indexableFieldDef = (IndexableFieldDef) fieldDefEntry.getValue();
           if (indexableFieldDef.hasDocValues()) {
@@ -951,6 +962,132 @@ private static void fetchFromValueSource(
       }
     }
 
+    /** Fetch field value from runtime field's Object. */
+    private static void fetchRuntimeFromSegmentFactory(
+        List<SearchResponse.Hit.Builder> sliceHits,
+        LeafReaderContext sliceSegment,
+        String name,
+        RuntimeFieldDef runtimeFieldDef)
+        throws IOException {
+      RuntimeScript.SegmentFactory segmentFactory = runtimeFieldDef.getSegmentFactory();
+      RuntimeScript values = segmentFactory.newInstance(sliceSegment);
+
+      for (SearchResponse.Hit.Builder hit : sliceHits) {
+        int docID = hit.getLuceneDocId() - sliceSegment.docBase;
+        // Check if the value is available for the current document
+        if (values != null) {
+          values.setDocId(docID);
+          Object obj = values.execute();
+          SearchResponse.Hit.CompositeFieldValue.Builder compositeFieldValue =
+              SearchResponse.Hit.CompositeFieldValue.newBuilder();
+          if (obj instanceof Float) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder().setFloatValue((Float) obj));
+          } else if (obj instanceof String) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder().setTextValue(String.valueOf(obj)));
+          } else if (obj instanceof Double) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder().setDoubleValue((Double) obj));
+          } else if (obj instanceof Long) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder().setLongValue((Long) obj));
+          } else if (obj instanceof Integer) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder().setIntValue((Integer) obj));
+          } else if (obj instanceof Map) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder()
+                    .setStructValue(createStruct((Map<String, Object>) obj)));
+          } else if (obj instanceof Boolean) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder().setBooleanValue((Boolean) obj));
+          } else if (obj instanceof ArrayList) {
+            SearchResponse.Hit.FieldValue.Builder fieldValueBuilder =
+                SearchResponse.Hit.FieldValue.newBuilder();
+            SearchResponse.Hit.FieldValue.RepeatedFieldValues.Builder repeatedFieldValuesBuilder =
+                SearchResponse.Hit.FieldValue.RepeatedFieldValues.newBuilder();
+            generateRepeatedFields(repeatedFieldValuesBuilder, obj);
+            fieldValueBuilder.setRepeatedFieldValues(repeatedFieldValuesBuilder);
+            compositeFieldValue.addFieldValue(fieldValueBuilder);
+          }
+          hit.putFields(name, compositeFieldValue.build());
+        }
+      }
+    }
+
+    /**
+     * Generates list fields for all types except for lists.
+     *
+     * <p>TODO: Support list of lists.
+     *
+     * @param repeatedFieldValuesBuilder The builder for repeated field value in the search
+     *     response.
+     * @param objs Runtime field objects that are supposed to be list.
+     */
+    private static void generateRepeatedFields(
+        SearchResponse.Hit.FieldValue.RepeatedFieldValues.Builder repeatedFieldValuesBuilder,
+        Object objs) {
+      List<Struct> structObjs = new ArrayList<>();
+      for (Object obj : (Iterable<? extends Object>) objs) {
+        if (obj instanceof String) {
+          repeatedFieldValuesBuilder.addAllTextValues((Iterable<String>) objs);
+          return;
+        } else if (obj instanceof Integer) {
+          repeatedFieldValuesBuilder.addAllIntValues((Iterable<Integer>) objs);
+          return;
+        } else if (obj instanceof Double) {
+          repeatedFieldValuesBuilder.addAllDoubleValues((Iterable<Double>) objs);
+          return;
+        } else if (obj instanceof Float) {
+          repeatedFieldValuesBuilder.addAllFloatValues((Iterable<Float>) objs);
+          return;
+        } else if (obj instanceof Long) {
+          repeatedFieldValuesBuilder.addAllLongValues((Iterable<Long>) objs);
+          return;
+        } else if (obj instanceof Boolean) {
+          repeatedFieldValuesBuilder.addAllBooleanValues((Iterable<Boolean>) objs);
+          return;
+        } else if (obj instanceof Map) {
+          structObjs.add(createStruct((Map<String, Object>) obj));
+        } else {
+          return;
+        }
+      }
+
+      // Construct repeated structs.
+      if (Iterables.size(structObjs) > 0) {
+        repeatedFieldValuesBuilder.addAllStructValues(structObjs);
+      }
+    }
+
+    /**
+     * Create a struct for the given map based on object value type
+     *
+     * @param map The given map
+     * @return Struct The protobuf representation of the map using a struct object.
+     */
+    private static Struct createStruct(Map<String, Object> map) {
+      Builder struct = Struct.newBuilder();
+      for (Map.Entry<String, Object> entry : map.entrySet()) {
+        if (entry.getValue() instanceof Number) {
+          struct.putFields(
+              entry.getKey(),
+              Value.newBuilder().setNumberValue(((Number) entry.getValue()).doubleValue()).build());
+        } else if (entry.getValue() instanceof String) {
+          struct.putFields(
+              entry.getKey(), Value.newBuilder().setStringValue((String) entry.getValue()).build());
+        } else if (entry.getValue() instanceof Map) {
+          struct.putFields(
+              entry.getKey(),
+              Value.newBuilder()
+                  .setStructValue(createStruct((Map<String, Object>) entry.getValue()))
+                  .build());
+        }
+      }
+      return struct.build();
+    }
+
     /** Fetch field value from its doc value */
     private static void fetchFromDocVales(
         List<SearchResponse.Hit.Builder> sliceHits,

src/main/java/com/yelp/nrtsearch/server/luceneserver/facet/DrillSidewaysImpl.java

@@ -28,6 +28,7 @@
 import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.IntFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.LongFieldDef;
+import com.yelp.nrtsearch.server.luceneserver.field.RuntimeFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.field.VirtualFieldDef;
 import com.yelp.nrtsearch.server.luceneserver.script.FacetScript;
 import com.yelp.nrtsearch.server.luceneserver.script.ScriptService;
@@ -324,10 +325,11 @@ private static com.yelp.nrtsearch.server.grpc.FacetResult getFieldFacetResult(
     }
 
     FacetResult facetResult;
-    if (!(fieldDef instanceof IndexableFieldDef) && !(fieldDef instanceof VirtualFieldDef)) {
+    if (!(fieldDef instanceof IndexableFieldDef)
+        && !(fieldDef instanceof VirtualFieldDef || fieldDef instanceof RuntimeFieldDef)) {
       throw new IllegalArgumentException(
           String.format(
-              "field %s is neither a virtual field nor registered as an indexable field. Facets are supported only for these types",
+              "field %s is neither a virtual/runtime field nor registered as an indexable field. Facets are supported only for these types",
               fieldName));
     }
     if (!facet.getNumericRangeList().isEmpty()) {

src/main/java/com/yelp/nrtsearch/server/luceneserver/field/FieldDefCreator.java

@@ -55,6 +55,11 @@ public FieldDefCreator(LuceneServerConfiguration configuration) {
         (name, field) -> {
           throw new UnsupportedOperationException("Virtual fields should be created directly");
         });
+    register(
+        "RUNTIME",
+        (name, field) -> {
+          throw new UnsupportedOperationException("Runtime fields should be created directly");
+        });
     register("VECTOR", VectorFieldDef::new);
     register("CONTEXT_SUGGEST", ContextSuggestFieldDef::new);
   }

src/main/java/com/yelp/nrtsearch/server/luceneserver/field/RuntimeFieldDef.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 Yelp Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.yelp.nrtsearch.server.luceneserver.field;
+
+import com.yelp.nrtsearch.server.luceneserver.field.IndexableFieldDef.FacetValueType;
+import com.yelp.nrtsearch.server.luceneserver.script.RuntimeScript;
+
+/**
+ * Field definition for a runtime field. Runtime fields are able to produce a value for each given
+ * index document.
+ */
+public class RuntimeFieldDef extends FieldDef {
+  private final RuntimeScript.SegmentFactory segmentFactory;
+  private final IndexableFieldDef.FacetValueType facetValueType;
+
+  /**
+   * Field constructor.
+   *
+   * @param name name of field
+   * @param segmentFactory lucene value source used to produce field value from documents
+   */
+  public RuntimeFieldDef(String name, RuntimeScript.SegmentFactory segmentFactory) {
+    super(name);
+    this.segmentFactory = segmentFactory;
+    this.facetValueType = FacetValueType.NO_FACETS;
+  }
+
+  /**
+   * Get value source for this field.
+   *
+   * @return Segment factory to create the expression.
+   */
+  public RuntimeScript.SegmentFactory getSegmentFactory() {
+    return segmentFactory;
+  }
+
+  @Override
+  public String getType() {
+    return "RUNTIME";
+  }
+
+  /**
+   * Get the facet value type for this field.
+   *
+   * @return field facet value type
+   */
+  @Override
+  public IndexableFieldDef.FacetValueType getFacetValueType() {
+    return facetValueType;
+  }
+}

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/FieldUpdateHandler.java

@@ -91,11 +91,14 @@ public static UpdatedFieldInfo updateFields(
     FieldAndFacetState.Builder fieldStateBuilder = currentState.toBuilder();
     List<Field> nonVirtualFields = new ArrayList<>();
     List<Field> virtualFields = new ArrayList<>();
+    List<Field> runtimeFields = new ArrayList<>();
 
     for (Field field : updateFields) {
       checkFieldName(field.getName());
       if (FieldType.VIRTUAL.equals(field.getType())) {
         virtualFields.add(field);
+      } else if (FieldType.RUNTIME.equals(field.getType())) {
+        runtimeFields.add(field);
       } else {
         nonVirtualFields.add(field);
       }
@@ -119,6 +122,7 @@ public static UpdatedFieldInfo updateFields(
       parseVirtualField(field, fieldStateBuilder);
       newFields.put(field.getName(), field);
     }
+
     return new UpdatedFieldInfo(newFields, fieldStateBuilder.build());
   }