improve ordering in a PathPlanner

fix several volatile tests caused by it
tribbloid · Feb 8, 2024 · 5800f96 · 5800f96
1 parent 8a8feff
commit 5800f96
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 25 deletions.
diff --git a/parent/core/src/main/scala/com/tribbloids/spookystuff/dsl/PathPlanners_Simple.scala b/parent/core/src/main/scala/com/tribbloids/spookystuff/dsl/PathPlanners_Simple.scala
@@ -36,7 +36,7 @@ object PathPlanners_Simple {
 
           val result = candidates
             .minBy { v =>
-              val indices = v.map(_.sortIndex(depthField, ordinalField))
+              val indices = v.map(_.sortIndex(schema.sortIndices: _*))
               indices
             }
 
@@ -58,7 +58,7 @@ object PathPlanners_Simple {
           val dataRows = tuple._2
           val firstDataRow = dataRows.head
 
-          val sortEvs = firstDataRow.sortIndex(depthField, ordinalField)
+          val sortEvs = firstDataRow.sortIndex(schema.sortIndices: _*)
           0 -> sortEvs
         }
         result

diff --git a/parent/core/src/main/scala/com/tribbloids/spookystuff/rdd/FetchedDataset.scala b/parent/core/src/main/scala/com/tribbloids/spookystuff/rdd/FetchedDataset.scala
@@ -368,7 +368,7 @@ case class FetchedDataset(
       traces: HasTraceSet,
       keyBy: Trace => Any = identity,
       genPartitioner: GenPartitioner = spooky.conf.localityPartitioner,
-      depthField: Field = null,
+      depthField: Field = null, // TODO: Some of them has to be moved upwards
       range: Range = spooky.conf.exploreRange,
       pathPlanning: PathPlanning = spooky.conf.explorePathPlanning,
       epochSize: Int = spooky.conf.exploreEpochSize,

diff --git a/parent/core/src/main/scala/com/tribbloids/spookystuff/row/SpookySchema.scala b/parent/core/src/main/scala/com/tribbloids/spookystuff/row/SpookySchema.scala
@@ -39,15 +39,15 @@ case class SpookySchema(
 
 //  lazy val evictTransientFields: SpookySchema = filterFields(_.isNonTransient)
 
-  lazy val structFields: Seq[StructField] = fieldTypes.toSeq
+  @transient lazy val structFields: Seq[StructField] = fieldTypes.toSeq
     .map { tuple =>
       StructField(
         tuple._1.name,
         tuple._2.reified
       )
     }
 
-  lazy val structType: StructType = {
+  @transient lazy val structType: StructType = {
 
     StructType(structFields)
   }
@@ -56,6 +56,8 @@ case class SpookySchema(
     fieldTypes = fieldTypes -- field
   )
 
+  @transient lazy val sortIndices: List[Field] = fields.filter(_.isSortIndex)
+
   def newResolver: Resolver = new Resolver()
 
   class Resolver extends Serializable {

diff --git a/parent/core/src/test/scala/com/tribbloids/spookystuff/execution/ExplorePlanSpec.scala b/parent/core/src/test/scala/com/tribbloids/spookystuff/execution/ExplorePlanSpec.scala
@@ -1,5 +1,6 @@
 package com.tribbloids.spookystuff.execution
 
+import ai.acyclic.prover.commons.debug.print_@
 import ai.acyclic.prover.commons.function.Impl
 import com.tribbloids.spookystuff.actions.{Trace, Wget}
 import com.tribbloids.spookystuff.extractors.impl.Lit
@@ -31,7 +32,7 @@ class ExplorePlanSpec extends SpookyBaseSpec with LocalPathDocsFixture {
         S"h1".text ~ 'header
       )
 
-    println(explored.plan.toString)
+    print_@(explored.plan.toString)
   }
 
   it("should create a new beaconRDD if its upstream doesn't have one") {
@@ -92,12 +93,14 @@ class ExplorePlanSpec extends SpookyBaseSpec with LocalPathDocsFixture {
         .fetch {
           Wget('_)
         }
-        .explore(S"root directory URI".text)(
+        .explore(
+          S"root directory URI".text
+        )(
           Wget('A),
           depthField = 'depth,
           range = range
         )
-        .fork(S"root file")(
+        .fork(S"root file", ordinalField = 'index)(
           A"name".text into 'leaf,
           A"URI".text ~ 'fullPath
         )
@@ -113,32 +116,26 @@ class ExplorePlanSpec extends SpookyBaseSpec with LocalPathDocsFixture {
 
     lazy val `-1..` = computeFrom(-1 to bigInt)
 
-    it("from 0") {
+    it("from -1") {
 
-      `0..`.mkString("\n")
+      `-1..`.mkString("\n")
         .shouldBe(
           """
-            |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(hivetable.csv),file:///tmp/spookystuff/resources/testutils/dir/hivetable.csv]
-            |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(table.csv),file:///tmp/spookystuff/resources/testutils/dir/table.csv]
-            |[/tmp/spookystuff/resources/testutils/dir,1,ArraySeq(hivetable.csv, Test.pdf),file:///tmp/spookystuff/resources/testutils/dir/dir/Test.pdf]
-            |[/tmp/spookystuff/resources/testutils/dir,2,ArraySeq(hivetable.csv, Test.pdf, pom.xml),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/pom.xml]
-            |[/tmp/spookystuff/resources/testutils/dir,3,ArraySeq(hivetable.csv, Test.pdf, pom.xml, tribbloid.json),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/dir/tribbloid.json]
+            |[/tmp/spookystuff/resources/testutils/dir,null,null,null,null]
+            |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(0),ArraySeq(hivetable.csv),file:///tmp/spookystuff/resources/testutils/dir/hivetable.csv]
+            |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(1),ArraySeq(table.csv),file:///tmp/spookystuff/resources/testutils/dir/table.csv]
+            |[/tmp/spookystuff/resources/testutils/dir,1,ArraySeq(0, 0),ArraySeq(hivetable.csv, Test.pdf),file:///tmp/spookystuff/resources/testutils/dir/dir/Test.pdf]
+            |[/tmp/spookystuff/resources/testutils/dir,2,ArraySeq(0, 0, 0),ArraySeq(hivetable.csv, Test.pdf, pom.xml),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/pom.xml]
+            |[/tmp/spookystuff/resources/testutils/dir,3,ArraySeq(0, 0, 0, 0),ArraySeq(hivetable.csv, Test.pdf, pom.xml, tribbloid.json),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/dir/tribbloid.json]
             |""".stripMargin
         )
     }
 
-    it("from -1") {
+    it("from 0") {
 
-      `-1..`.mkString("\n")
+      `0..`.mkString("\n")
         .shouldBe(
-          """
-            |[/tmp/spookystuff/resources/testutils/dir,null,null,null]
-            |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(hivetable.csv),file:///tmp/spookystuff/resources/testutils/dir/hivetable.csv]
-            |[/tmp/spookystuff/resources/testutils/dir,0,ArraySeq(table.csv),file:///tmp/spookystuff/resources/testutils/dir/table.csv]
-            |[/tmp/spookystuff/resources/testutils/dir,1,ArraySeq(hivetable.csv, Test.pdf),file:///tmp/spookystuff/resources/testutils/dir/dir/Test.pdf]
-            |[/tmp/spookystuff/resources/testutils/dir,2,ArraySeq(hivetable.csv, Test.pdf, pom.xml),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/pom.xml]
-            |[/tmp/spookystuff/resources/testutils/dir,3,ArraySeq(hivetable.csv, Test.pdf, pom.xml, tribbloid.json),file:///tmp/spookystuff/resources/testutils/dir/dir/dir/dir/tribbloid.json]
-            |""".stripMargin
+          `-1..`.slice(1, bigInt).mkString("\n")
         )
     }