Use json util object converter and address other commends

clientlib/src/main/java/com/yelp/nrtsearch/server/utils/StructJsonUtils.java

@@ -71,6 +71,22 @@ public static Struct convertMapToStruct(Map<String, Object> map, boolean longAsS
     return builder.build();
   }
 
+  /**
+   * Convert an Iterable of native java types into a protobuf ListValue. This iterable may contain null,
+   * Boolean, String, Number, Iterable, or Map (String key).
+   *
+   * @param iterable iterable of java native types
+   * @param longAsString if Long values should be encoded as String
+   * @return protobuf list value for iterable
+   */
+  public static ListValue convertIterableToListValue(Iterable<?> iterable, boolean longAsString) {
+    List<Value> valueList = new ArrayList<>();
+    for (Object e : iterable) {
+      valueList.add(convertObjectToValue(e, longAsString));
+    }
+    return ListValue.newBuilder().addAllValues(valueList).build();
+  }
+
   /**
    * Convert an Iterable of native java types into a protobuf Value. This iterable may contain null,
    * Boolean, String, Number, Iterable, or Map (String key).

clientlib/src/main/proto/yelp/nrtsearch/search.proto

@@ -608,30 +608,12 @@ message SearchResponse {
                 google.protobuf.Struct structValue = 8; // Value for structured data
                 // Value for VECTOR FieldType
                 Vector vectorValue = 9;
-                RepeatedFieldValues repeatedFieldValues = 10;
+                google.protobuf.ListValue listValue = 10;
             }
 
             message Vector {
                 repeated float value = 1;
             }
-
-            message RepeatedFieldValues {
-                // Repeated value for ATOM and TEXT  FieldType
-                repeated string textValues = 1; 
-                // Repeated value for BOOLEAN FieldType
-                repeated bool booleanValues = 2;
-                // Repeated value for INT FieldType2; 
-                repeated int32 intValues = 3; 
-                // Repeated value for LONG and DATE_TIME (as milliseconds since epoch in UTC zone) FieldType
-                repeated int64 longValues = 4; 
-                // Repeated value for FLOAT FieldType
-                repeated float floatValues = 5; 
-                // Repeated value for DOUBLE FieldType
-                repeated double doubleValues = 6; 
-                // Repeated value for structured data
-                repeated google.protobuf.Struct structValues = 7; 
-            }
-
         }
 
         message CompositeFieldValue {

src/main/java/com/yelp/nrtsearch/server/luceneserver/SearchHandler.java

@@ -15,11 +15,8 @@
  */
 package com.yelp.nrtsearch.server.luceneserver;
 
-import com.google.common.collect.Iterables;
 import com.google.common.collect.Lists;
 import com.google.protobuf.Struct;
-import com.google.protobuf.Struct.Builder;
-import com.google.protobuf.Value;
 import com.google.protobuf.util.JsonFormat;
 import com.yelp.nrtsearch.server.grpc.DeadlineUtils;
 import com.yelp.nrtsearch.server.grpc.FacetResult;
@@ -51,6 +48,7 @@
 import com.yelp.nrtsearch.server.luceneserver.search.SearchRequestProcessor;
 import com.yelp.nrtsearch.server.luceneserver.search.SearcherResult;
 import com.yelp.nrtsearch.server.monitoring.SearchResponseCollector;
+import com.yelp.nrtsearch.server.utils.StructJsonUtils;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Comparator;
@@ -998,96 +996,21 @@ private static void fetchRuntimeFromSegmentFactory(
           } else if (obj instanceof Map) {
             compositeFieldValue.addFieldValue(
                 SearchResponse.Hit.FieldValue.newBuilder()
-                    .setStructValue(createStruct((Map<String, Object>) obj)));
+                    .setStructValue(StructJsonUtils.convertMapToStruct((Map<String, Object>) obj)));
           } else if (obj instanceof Boolean) {
             compositeFieldValue.addFieldValue(
                 SearchResponse.Hit.FieldValue.newBuilder().setBooleanValue((Boolean) obj));
-          } else if (obj instanceof ArrayList) {
-            SearchResponse.Hit.FieldValue.Builder fieldValueBuilder =
-                SearchResponse.Hit.FieldValue.newBuilder();
-            SearchResponse.Hit.FieldValue.RepeatedFieldValues.Builder repeatedFieldValuesBuilder =
-                SearchResponse.Hit.FieldValue.RepeatedFieldValues.newBuilder();
-            generateRepeatedFields(repeatedFieldValuesBuilder, obj);
-            fieldValueBuilder.setRepeatedFieldValues(repeatedFieldValuesBuilder);
-            compositeFieldValue.addFieldValue(fieldValueBuilder);
+          } else if (obj instanceof Iterable<?>) {
+            compositeFieldValue.addFieldValue(
+                SearchResponse.Hit.FieldValue.newBuilder()
+                    .setListValue(
+                        StructJsonUtils.convertIterableToListValue((Iterable<?>) obj, false)));
           }
           hit.putFields(name, compositeFieldValue.build());
         }
       }
     }
 
-    /**
-     * Generates list fields for all types except for lists.
-     *
-     * <p>TODO: Support list of lists.
-     *
-     * @param repeatedFieldValuesBuilder The builder for repeated field value in the search
-     *     response.
-     * @param objs Runtime field objects that are supposed to be list.
-     */
-    private static void generateRepeatedFields(
-        SearchResponse.Hit.FieldValue.RepeatedFieldValues.Builder repeatedFieldValuesBuilder,
-        Object objs) {
-      List<Struct> structObjs = new ArrayList<>();
-      for (Object obj : (Iterable<? extends Object>) objs) {
-        if (obj instanceof String) {
-          repeatedFieldValuesBuilder.addAllTextValues((Iterable<String>) objs);
-          return;
-        } else if (obj instanceof Integer) {
-          repeatedFieldValuesBuilder.addAllIntValues((Iterable<Integer>) objs);
-          return;
-        } else if (obj instanceof Double) {
-          repeatedFieldValuesBuilder.addAllDoubleValues((Iterable<Double>) objs);
-          return;
-        } else if (obj instanceof Float) {
-          repeatedFieldValuesBuilder.addAllFloatValues((Iterable<Float>) objs);
-          return;
-        } else if (obj instanceof Long) {
-          repeatedFieldValuesBuilder.addAllLongValues((Iterable<Long>) objs);
-          return;
-        } else if (obj instanceof Boolean) {
-          repeatedFieldValuesBuilder.addAllBooleanValues((Iterable<Boolean>) objs);
-          return;
-        } else if (obj instanceof Map) {
-          structObjs.add(createStruct((Map<String, Object>) obj));
-        } else {
-          return;
-        }
-      }
-
-      // Construct repeated structs.
-      if (Iterables.size(structObjs) > 0) {
-        repeatedFieldValuesBuilder.addAllStructValues(structObjs);
-      }
-    }
-
-    /**
-     * Create a struct for the given map based on object value type
-     *
-     * @param map The given map
-     * @return Struct The protobuf representation of the map using a struct object.
-     */
-    private static Struct createStruct(Map<String, Object> map) {
-      Builder struct = Struct.newBuilder();
-      for (Map.Entry<String, Object> entry : map.entrySet()) {
-        if (entry.getValue() instanceof Number) {
-          struct.putFields(
-              entry.getKey(),
-              Value.newBuilder().setNumberValue(((Number) entry.getValue()).doubleValue()).build());
-        } else if (entry.getValue() instanceof String) {
-          struct.putFields(
-              entry.getKey(), Value.newBuilder().setStringValue((String) entry.getValue()).build());
-        } else if (entry.getValue() instanceof Map) {
-          struct.putFields(
-              entry.getKey(),
-              Value.newBuilder()
-                  .setStructValue(createStruct((Map<String, Object>) entry.getValue()))
-                  .build());
-        }
-      }
-      return struct.build();
-    }
-
     /** Fetch field value from its doc value */
     private static void fetchFromDocVales(
         List<SearchResponse.Hit.Builder> sliceHits,

src/main/java/com/yelp/nrtsearch/server/luceneserver/index/handlers/FieldUpdateHandler.java

@@ -91,14 +91,11 @@ public static UpdatedFieldInfo updateFields(
     FieldAndFacetState.Builder fieldStateBuilder = currentState.toBuilder();
     List<Field> nonVirtualFields = new ArrayList<>();
     List<Field> virtualFields = new ArrayList<>();
-    List<Field> runtimeFields = new ArrayList<>();
 
     for (Field field : updateFields) {
       checkFieldName(field.getName());
       if (FieldType.VIRTUAL.equals(field.getType())) {
         virtualFields.add(field);
-      } else if (FieldType.RUNTIME.equals(field.getType())) {
-        runtimeFields.add(field);
       } else {
         nonVirtualFields.add(field);
       }

src/main/java/com/yelp/nrtsearch/server/luceneserver/script/RuntimeScript.java

@@ -70,11 +70,7 @@ public Map<String, LoadedDocValues<?>> getDoc() {
     return segmentDocLookup;
   }
 
-  /**
-   * Factory interface for creating a
-   * RuntimeSsrc/main/java/com/yelp/nrtsearch/server/luceneserver/script/RuntimeScript.java cript
-   * bound to a lucene segment.
-   */
+  /** Factory interface for creating a RuntimeScript bound to a lucene segment. */
   public interface SegmentFactory {
 
     /**

src/test/java/com/yelp/nrtsearch/server/luceneserver/facet/RuntimeScriptFacetsTest.java

@@ -39,7 +39,7 @@
 
 public class RuntimeScriptFacetsTest extends ServerTestCase {
   private static final int NUM_DOCS = 100;
-  private static final int SEGMENT_CHUNK = 10;
+  private static final int TOP_HITS = 10;
 
   @ClassRule public static final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
 
@@ -53,7 +53,7 @@ public static class TestRuntimeScriptEngine implements ScriptEngine {
 
       @Override
       public String getLang() {
-        return "painless";
+        return "test_p";
       }
 
       @Override
@@ -179,7 +179,7 @@ protected List<String> getIndices() {
   }
 
   protected FieldDefRequest getIndexDef(String name) throws IOException {
-    return getFieldsFromResourceFile("/facet/facet_script_facets.json");
+    return getFieldsFromResourceFile("/facet/runtime_field_script.json");
   }
 
   protected void initIndex(String name) throws Exception {
@@ -215,7 +215,7 @@ protected void initIndex(String name) throws Exception {
                       .build())
               .build());
 
-      if (requestChunk.size() == SEGMENT_CHUNK) {
+      if (requestChunk.size() == TOP_HITS) {
         addDocuments(requestChunk.stream());
         requestChunk.clear();
         writer.commit();
@@ -233,17 +233,17 @@ public void RuntimeScriptForInt() {
 
     RuntimeField runtimeField =
         RuntimeField.newBuilder()
-            .setScript(Script.newBuilder().setLang("painless").setSource("int").build())
+            .setScript(Script.newBuilder().setLang("test_p").setSource("int").build())
             .setName("runtime_field")
             .build();
 
     List expectedValues = new ArrayList<>();
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    for (int id = 0; id < TOP_HITS; ++id) {
       expectedValues.add(2);
     }
     SearchResponse response = doQuery(runtimeField);
-    assertEquals(SEGMENT_CHUNK, response.getHitsCount());
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    assertEquals(TOP_HITS, response.getHitsCount());
+    for (int id = 0; id < TOP_HITS; ++id) {
       assertEquals(
           response.getHits(id).getFieldsMap().get("runtime_field").getFieldValue(0).getIntValue(),
           expectedValues.get(id));
@@ -255,17 +255,17 @@ public void RuntimeScriptForString() {
 
     RuntimeField runtimeField =
         RuntimeField.newBuilder()
-            .setScript(Script.newBuilder().setLang("painless").setSource("string").build())
+            .setScript(Script.newBuilder().setLang("test_p").setSource("string").build())
             .setName("runtime_field")
             .build();
 
     List expectedValues = new ArrayList<>();
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    for (int id = 0; id < TOP_HITS; ++id) {
       expectedValues.add("2");
     }
     SearchResponse response = doQuery(runtimeField);
-    assertEquals(SEGMENT_CHUNK, response.getHitsCount());
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    assertEquals(TOP_HITS, response.getHitsCount());
+    for (int id = 0; id < TOP_HITS; ++id) {
       assertEquals(
           response.getHits(id).getFieldsMap().get("runtime_field").getFieldValue(0).getTextValue(),
           expectedValues.get(id));
@@ -277,17 +277,17 @@ public void RuntimeScriptForMap() {
 
     RuntimeField runtimeField =
         RuntimeField.newBuilder()
-            .setScript(Script.newBuilder().setLang("painless").setSource("map").build())
+            .setScript(Script.newBuilder().setLang("test_p").setSource("map").build())
             .setName("runtime_field")
             .build();
 
     List expectedValues = new ArrayList<>();
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    for (int id = 0; id < TOP_HITS; ++id) {
       expectedValues.add(2.0);
     }
     SearchResponse response = doQuery(runtimeField);
-    assertEquals(SEGMENT_CHUNK, response.getHitsCount());
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    assertEquals(TOP_HITS, response.getHitsCount());
+    for (int id = 0; id < TOP_HITS; ++id) {
       assertEquals(
           response
               .getHits(id)
@@ -307,20 +307,20 @@ public void RuntimeScriptForList() {
 
     RuntimeField runtimeField =
         RuntimeField.newBuilder()
-            .setScript(Script.newBuilder().setLang("painless").setSource("list").build())
+            .setScript(Script.newBuilder().setLang("test_p").setSource("list").build())
             .setName("runtime_field")
             .build();
 
     List<List<String>> expectedValues = new ArrayList<>();
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    for (int id = 0; id < TOP_HITS; ++id) {
       List nums = new ArrayList();
       nums.add("1");
       nums.add("2");
       expectedValues.add(nums);
     }
     SearchResponse response = doQuery(runtimeField);
-    assertEquals(SEGMENT_CHUNK, response.getHitsCount());
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    assertEquals(TOP_HITS, response.getHitsCount());
+    for (int id = 0; id < TOP_HITS; ++id) {
       assertEquals(
           response
               .getHits(id)
@@ -339,17 +339,17 @@ public void RuntimeScriptForDocValue() {
 
     RuntimeField runtimeField =
         RuntimeField.newBuilder()
-            .setScript(Script.newBuilder().setLang("painless").setSource("docValue").build())
+            .setScript(Script.newBuilder().setLang("test_p").setSource("docValue").build())
             .setName("runtime_field")
             .build();
 
     List expectedValues = new ArrayList<>();
-    for (int id = 0; id < SEGMENT_CHUNK; ++id) {
+    for (int id = 0; id < TOP_HITS; ++id) {
       expectedValues.add(String.valueOf(id % 3) + "_" + String.valueOf(id % 2));
     }
     SearchResponse response = doQuery(runtimeField);
-    assertEquals(SEGMENT_CHUNK, response.getHitsCount());
-    for (int id = 0; id < SEGMENT_CHUNK; id++) {
+    assertEquals(TOP_HITS, response.getHitsCount());
+    for (int id = 0; id < TOP_HITS; id++) {
       assertEquals(
           response.getHits(id).getFieldsMap().get("runtime_field").getFieldValue(0).getTextValue(),
           expectedValues.get(id));