From 83cef51e1c7f65c2a57a6117964d06eaa314500a Mon Sep 17 00:00:00 2001 From: Nicolas Stucki Date: Mon, 28 May 2018 14:41:45 +0200 Subject: [PATCH 01/28] Fix #4591: Unpickle type quotes with TreeReader.readTpt --- .../dotc/core/quoted/PickledQuotes.scala | 27 +++++++++++-------- .../dotc/core/tasty/DottyUnpickler.scala | 3 +++ .../tools/dotc/core/tasty/TreeUnpickler.scala | 7 +++++ tests/run-with-compiler/i4591.check | 1 + tests/run-with-compiler/i4591.scala | 15 +++++++++++ 5 files changed, 42 insertions(+), 11 deletions(-) create mode 100644 tests/run-with-compiler/i4591.check create mode 100644 tests/run-with-compiler/i4591.scala diff --git a/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala b/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala index faf50b5848a6..007e47e30a5f 100644 --- a/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala +++ b/compiler/src/dotty/tools/dotc/core/quoted/PickledQuotes.scala @@ -66,13 +66,13 @@ object PickledQuotes { /** Unpickle the tree contained in the TastyExpr */ private def unpickleExpr(expr: TastyExpr[_])(implicit ctx: Context): Tree = { val tastyBytes = TastyString.unpickle(expr.tasty) - unpickle(tastyBytes, expr.args) + unpickle(tastyBytes, expr.args, isType = false) } /** Unpickle the tree contained in the TastyType */ private def unpickleType(ttpe: TastyType[_])(implicit ctx: Context): Tree = { val tastyBytes = TastyString.unpickle(ttpe.tasty) - unpickle(tastyBytes, ttpe.args) + unpickle(tastyBytes, ttpe.args, isType = true) } // TASTY picklingtests/pos/quoteTest.scala @@ -85,28 +85,33 @@ object PickledQuotes { treePkl.compactify() pickler.addrOfTree = treePkl.buf.addrOfTree pickler.addrOfSym = treePkl.addrOfSym - // if (tree.pos.exists) - // new PositionPickler(pickler, treePkl.buf.addrOfTree).picklePositions(tree :: Nil) - // other pickle sections go here. + if (pickling ne noPrinter) + println(i"**** pickling quote of \n${tree.show}") + val pickled = pickler.assembleParts() - if (pickling ne noPrinter) { - println(i"**** pickled quote of \n${tree.show}") + if (pickling ne noPrinter) new TastyPrinter(pickled).printContents() - } pickled } /** Unpickle TASTY bytes into it's tree */ - private def unpickle(bytes: Array[Byte], splices: Seq[Any])(implicit ctx: Context): Tree = { + private def unpickle(bytes: Array[Byte], splices: Seq[Any], isType: Boolean)(implicit ctx: Context): Tree = { val unpickler = new TastyUnpickler(bytes, splices) - val tree = unpickler.unpickleExpr() if (pickling ne noPrinter) { - println(i"**** unpickled quote for \n${tree.show}") + println(i"**** unpickling quote from TASTY") new TastyPrinter(bytes).printContents() } + + val tree = + if (isType) unpickler.unpickleTypeTree() + else unpickler.unpickleExpr() + + if (pickling ne noPrinter) + println(i"**** unpickle quote ${tree.show}") + tree } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala index 5dc92ee916df..91832b997877 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala @@ -49,6 +49,9 @@ class DottyUnpickler(bytes: Array[Byte]) extends ClassfileParser.Embedded with t def unpickleExpr()(implicit ctx: Context): Tree = treeUnpickler.unpickleExpr() + def unpickleTypeTree()(implicit ctx: Context): Tree = + treeUnpickler.unpickleTypeTree() + protected def treeSectionUnpickler(posUnpicklerOpt: Option[PositionUnpickler]): TreeSectionUnpickler = { new TreeSectionUnpickler(posUnpicklerOpt) } diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala index 1a8fd49a5d99..5b35002530f7 100644 --- a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala +++ b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala @@ -85,6 +85,13 @@ class TreeUnpickler(reader: TastyReader, rdr.readTerm() } + def unpickleTypeTree()(implicit ctx: Context): Tree = { + this.roots = Set(ctx.owner) + val rdr = new TreeReader(reader).fork + ownerTree = new OwnerTree(NoAddr, 0, rdr.fork, reader.endAddr) + rdr.readTpt() + } + /** The unpickled trees */ def unpickle()(implicit ctx: Context): List[Tree] = { assert(roots != null, "unpickle without previous enterTopLevel") diff --git a/tests/run-with-compiler/i4591.check b/tests/run-with-compiler/i4591.check new file mode 100644 index 000000000000..ee1eff6a5131 --- /dev/null +++ b/tests/run-with-compiler/i4591.check @@ -0,0 +1 @@ +Some(9) diff --git a/tests/run-with-compiler/i4591.scala b/tests/run-with-compiler/i4591.scala new file mode 100644 index 000000000000..ad5119618c96 --- /dev/null +++ b/tests/run-with-compiler/i4591.scala @@ -0,0 +1,15 @@ +import dotty.tools.dotc.quoted.Toolbox._ +import scala.quoted._ + +object Test { + + def foo[T: Type](init: Expr[T]): Expr[Unit] = '{ + var x = ~init + println(x) + } + + def main(args: Array[String]): Unit = { + foo('(Option(9))).run + } + +} From 8c97458d08b3825c2ad5f52aaba2668b13719884 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Fri, 23 Mar 2018 15:26:02 +0100 Subject: [PATCH 02/28] Begin porting strymonas --- tests/run-with-compiler/staged-streams.scala | 111 +++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 tests/run-with-compiler/staged-streams.scala diff --git a/tests/run-with-compiler/staged-streams.scala b/tests/run-with-compiler/staged-streams.scala new file mode 100644 index 000000000000..4504b87610bb --- /dev/null +++ b/tests/run-with-compiler/staged-streams.scala @@ -0,0 +1,111 @@ +import dotty.tools.dotc.quoted.Toolbox._ +import scala.quoted._ + +trait StagedStreams { + + // TODO: remove as it exists in Quoted Lib + sealed trait Var[T] { + def get: Expr[T] + def update(x: Expr[T]): Expr[Unit] + } + + object Var { + def apply[T: Type, U](init: Expr[T])(body: Var[T] => Expr[U]): Expr[U] = '{ + var x = ~init + ~body( + new Var[T] { + def get: Expr[T] = '(x) + def update(e: Expr[T]): Expr[Unit] = '{ x = ~e } + } + ) + } + } + + type Id[A] = A + + trait Producer[A] { self => + type St + val card: Cardinality + + def init(k: St => Expr[Unit]): Expr[Unit] + def step(st: St, k: (A => Expr[Unit])): Expr[Unit] + def hasNext(st: St): Expr[Boolean] + } + + trait Cardinality + case object AtMost1 extends Cardinality + case object Many extends Cardinality + + trait StagedStream[A] + case class Linear[A](producer: Producer[A]) extends StagedStream[A] + case class Nested[A, B](producer: Producer[A], nestedf: A => StagedStream[B]) extends StagedStream[B] + + case class Stream[A](stream: StagedStream[Expr[A]]) { + // def fold[W](z: Expr[W], f: (Expr[W] => Expr[A] => Expr[W])): Expr[W] = ??? + + // def fold_raw[W](z: Expr[W], update_acc: Expr[W] => Expr[Unit], f: (Expr[W] => Expr[A] => Expr[W])): Expr[Unit] = { + // def consume[A](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { + // stream match { + // case Linear(producer) => { + // producer.card match { + // case Many => + // producer.init(sp => '{ + // while(~producer.hasNext(sp)) { + // ~producer.step(sp, consumer) + // } + // }) + // case AtMost1 => + // producer.init(sp => '{ + // if (~producer.hasNext(sp)) { + // ~producer.step(sp, consumer) + // } + // }) + // } + // } + // case Nested(producer, nestedf) => { + // ??? //consume(((a) => consume(consumer, nestedf(a))), Linear[A](producer)) + // } + // } + // } + + // ??? // consume((a: Expr[A]) => '{ ~update_acc(f(z)(a)) }, stream) + // } + } + + object Stream { + def of[A: Type](arr: Expr[Array[A]]): Stream[A] = { + val prod = new Producer[Expr[A]] { + type St = (Var[Int], Var[Int], Expr[Array[A]]) + + val card = Many + + def init(k: St => Expr[Unit]): Expr[Unit] = { + Var('{(~arr).length}) { n => + Var(0.toExpr){i => + k((i, n, arr)) + } + } + } + + def step(st: St, k: (Expr[A] => Expr[Unit])): Expr[Unit] = { + val (i, _, arr) = st + '{ + val el = (~arr).apply(~i.get) + ~i.update('{ ~i.get + 1 }) + ~k('(el)) + } + } + + def hasNext(st: St): Expr[Boolean] = { + val (i, n, _) = st + '{ + (~i.get < ~n.get) + } + } + } + + Stream(Linear(prod)) + } + } +} + From f75c2141ee0adfdc87b4345dca6f2e6d3339ddfd Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Fri, 6 Apr 2018 17:56:29 +0200 Subject: [PATCH 03/28] Restructuring --- .../staged-streams/staged-streams_1.scala} | 6 +++++- tests/pos/staged-streams/staged-streams_2.scala | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) rename tests/{run-with-compiler/staged-streams.scala => pos/staged-streams/staged-streams_1.scala} (98%) create mode 100644 tests/pos/staged-streams/staged-streams_2.scala diff --git a/tests/run-with-compiler/staged-streams.scala b/tests/pos/staged-streams/staged-streams_1.scala similarity index 98% rename from tests/run-with-compiler/staged-streams.scala rename to tests/pos/staged-streams/staged-streams_1.scala index 4504b87610bb..572efa88a91a 100644 --- a/tests/run-with-compiler/staged-streams.scala +++ b/tests/pos/staged-streams/staged-streams_1.scala @@ -1,7 +1,9 @@ +package streams + import dotty.tools.dotc.quoted.Toolbox._ import scala.quoted._ -trait StagedStreams { +object StagedStreams { // TODO: remove as it exists in Quoted Lib sealed trait Var[T] { @@ -109,3 +111,5 @@ trait StagedStreams { } } + + diff --git a/tests/pos/staged-streams/staged-streams_2.scala b/tests/pos/staged-streams/staged-streams_2.scala new file mode 100644 index 000000000000..f7d22a55562e --- /dev/null +++ b/tests/pos/staged-streams/staged-streams_2.scala @@ -0,0 +1,9 @@ +package streams + +import dotty.tools.dotc.quoted.Toolbox._ +import scala.quoted._ + +object Test { + + def test1() = StagedStreams.Stream.of('{Array(1,2,3)}) +} \ No newline at end of file From be16938dda5c7dfeff5588cc5de595658ef286c2 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Mon, 9 Apr 2018 17:47:16 +0200 Subject: [PATCH 04/28] Enable fold --- .../pos/staged-streams/staged-streams_1.scala | 63 ++++++++++--------- .../pos/staged-streams/staged-streams_2.scala | 10 ++- 2 files changed, 43 insertions(+), 30 deletions(-) diff --git a/tests/pos/staged-streams/staged-streams_1.scala b/tests/pos/staged-streams/staged-streams_1.scala index 572efa88a91a..02f36ffc2e66 100644 --- a/tests/pos/staged-streams/staged-streams_1.scala +++ b/tests/pos/staged-streams/staged-streams_1.scala @@ -43,35 +43,40 @@ object StagedStreams { case class Nested[A, B](producer: Producer[A], nestedf: A => StagedStream[B]) extends StagedStream[B] case class Stream[A](stream: StagedStream[Expr[A]]) { - // def fold[W](z: Expr[W], f: (Expr[W] => Expr[A] => Expr[W])): Expr[W] = ??? - - // def fold_raw[W](z: Expr[W], update_acc: Expr[W] => Expr[Unit], f: (Expr[W] => Expr[A] => Expr[W])): Expr[Unit] = { - // def consume[A](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { - // stream match { - // case Linear(producer) => { - // producer.card match { - // case Many => - // producer.init(sp => '{ - // while(~producer.hasNext(sp)) { - // ~producer.step(sp, consumer) - // } - // }) - // case AtMost1 => - // producer.init(sp => '{ - // if (~producer.hasNext(sp)) { - // ~producer.step(sp, consumer) - // } - // }) - // } - // } - // case Nested(producer, nestedf) => { - // ??? //consume(((a) => consume(consumer, nestedf(a))), Linear[A](producer)) - // } - // } - // } - - // ??? // consume((a: Expr[A]) => '{ ~update_acc(f(z)(a)) }, stream) - // } + def fold[W](z: Expr[W], f: ((Expr[W], Expr[A]) => Expr[W])): Expr[W] = { + Var('{z}) { s => + ~fold_raw((a: Expr[A]) => '{ + s.update(f(~s.get, a)) + s.get + }, stream) + acc + } + } + + def fold_raw[W](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { + stream match { + case Linear(producer) => { + producer.card match { + case Many => + producer.init(sp => '{ + while(~producer.hasNext(sp)) { + ~producer.step(sp, consumer) + } + }) + case AtMost1 => + producer.init(sp => '{ + if (~producer.hasNext(sp)) { + ~producer.step(sp, consumer) + } + }) + } + } + case _ => ??? +// Nested(producer, nestedf) => { +// ??? //consume(((a) => consume(consumer, nestedf(a))), Linear[A](producer)) +// } + } + } } object Stream { diff --git a/tests/pos/staged-streams/staged-streams_2.scala b/tests/pos/staged-streams/staged-streams_2.scala index f7d22a55562e..4edae132fb5d 100644 --- a/tests/pos/staged-streams/staged-streams_2.scala +++ b/tests/pos/staged-streams/staged-streams_2.scala @@ -5,5 +5,13 @@ import scala.quoted._ object Test { - def test1() = StagedStreams.Stream.of('{Array(1,2,3)}) + + def test1() = StagedStreams.Stream + .of('{Array(1,2,3)}) + .fold('{0}, ((a: Expr[Int])=> (b : Expr[Int]) => '{ ~a + ~b })) + + + def main(args: Array[String]): Unit = { + println(test1().show) + } } \ No newline at end of file From ff6809201946221c9804db4720ac0829efefa485 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 10 Apr 2018 20:16:21 +0200 Subject: [PATCH 05/28] Implement fold --- .../pos/staged-streams/staged-streams_2.scala | 17 ------- .../run-with-compiler/staged-streams_1.check | 1 + .../staged-streams_1.scala | 46 +++++++++++-------- 3 files changed, 29 insertions(+), 35 deletions(-) delete mode 100644 tests/pos/staged-streams/staged-streams_2.scala create mode 100644 tests/run-with-compiler/staged-streams_1.check rename tests/{pos/staged-streams => run-with-compiler}/staged-streams_1.scala (72%) diff --git a/tests/pos/staged-streams/staged-streams_2.scala b/tests/pos/staged-streams/staged-streams_2.scala deleted file mode 100644 index 4edae132fb5d..000000000000 --- a/tests/pos/staged-streams/staged-streams_2.scala +++ /dev/null @@ -1,17 +0,0 @@ -package streams - -import dotty.tools.dotc.quoted.Toolbox._ -import scala.quoted._ - -object Test { - - - def test1() = StagedStreams.Stream - .of('{Array(1,2,3)}) - .fold('{0}, ((a: Expr[Int])=> (b : Expr[Int]) => '{ ~a + ~b })) - - - def main(args: Array[String]): Unit = { - println(test1().show) - } -} \ No newline at end of file diff --git a/tests/run-with-compiler/staged-streams_1.check b/tests/run-with-compiler/staged-streams_1.check new file mode 100644 index 000000000000..62f9457511f8 --- /dev/null +++ b/tests/run-with-compiler/staged-streams_1.check @@ -0,0 +1 @@ +6 \ No newline at end of file diff --git a/tests/pos/staged-streams/staged-streams_1.scala b/tests/run-with-compiler/staged-streams_1.scala similarity index 72% rename from tests/pos/staged-streams/staged-streams_1.scala rename to tests/run-with-compiler/staged-streams_1.scala index 02f36ffc2e66..5f0a869353fc 100644 --- a/tests/pos/staged-streams/staged-streams_1.scala +++ b/tests/run-with-compiler/staged-streams_1.scala @@ -1,9 +1,7 @@ -package streams - import dotty.tools.dotc.quoted.Toolbox._ import scala.quoted._ -object StagedStreams { +object Test { // TODO: remove as it exists in Quoted Lib sealed trait Var[T] { @@ -40,21 +38,24 @@ object StagedStreams { trait StagedStream[A] case class Linear[A](producer: Producer[A]) extends StagedStream[A] - case class Nested[A, B](producer: Producer[A], nestedf: A => StagedStream[B]) extends StagedStream[B] + case class Nested[A, B](producer: Producer[B], nestedf: B => StagedStream[A]) extends StagedStream[A] case class Stream[A](stream: StagedStream[Expr[A]]) { - def fold[W](z: Expr[W], f: ((Expr[W], Expr[A]) => Expr[W])): Expr[W] = { - Var('{z}) { s => - ~fold_raw((a: Expr[A]) => '{ - s.update(f(~s.get, a)) - s.get - }, stream) - acc + + def fold[W: Type](z: Expr[W], f: ((Expr[W], Expr[A]) => Expr[W])): Expr[W] = { + Var(z) { s: Var[W] => '{ + + ~fold_raw[Expr[A]]((a: Expr[A]) => '{ + ~s.update(f(s.get, a)) + }, stream) + + ~s.get + } } } - def fold_raw[W](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { - stream match { + def fold_raw[A](consumer: A => Expr[Unit], s: StagedStream[A]): Expr[Unit] = { + s match { case Linear(producer) => { producer.card match { case Many => @@ -71,10 +72,10 @@ object StagedStreams { }) } } - case _ => ??? -// Nested(producer, nestedf) => { -// ??? //consume(((a) => consume(consumer, nestedf(a))), Linear[A](producer)) -// } + case nested: Nested[a, bt] => ??? +// { +// fold_raw[bt](((e: bt) => fold_raw(consumer, nested.nestedf(e))), Linear(nested.producer)) +// } } } } @@ -88,7 +89,7 @@ object StagedStreams { def init(k: St => Expr[Unit]): Expr[Unit] = { Var('{(~arr).length}) { n => - Var(0.toExpr){i => + Var(0.toExpr){ i => k((i, n, arr)) } } @@ -114,6 +115,15 @@ object StagedStreams { Stream(Linear(prod)) } } + + def test1() = Stream + .of('{Array(1, 2, 3)}) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + + + def main(args: Array[String]): Unit = { + println(test1().run) + } } From 99071e0a094f19c168de0aff0095d98b79cc516a Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 11 Apr 2018 15:37:36 +0200 Subject: [PATCH 06/28] Implement map --- .../run-with-compiler/staged-streams_1.check | 4 +- .../run-with-compiler/staged-streams_1.scala | 43 +++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/run-with-compiler/staged-streams_1.check b/tests/run-with-compiler/staged-streams_1.check index 62f9457511f8..71e62be89621 100644 --- a/tests/run-with-compiler/staged-streams_1.check +++ b/tests/run-with-compiler/staged-streams_1.check @@ -1 +1,3 @@ -6 \ No newline at end of file +6 + +12 \ No newline at end of file diff --git a/tests/run-with-compiler/staged-streams_1.scala b/tests/run-with-compiler/staged-streams_1.scala index 5f0a869353fc..3d20c90cfac3 100644 --- a/tests/run-with-compiler/staged-streams_1.scala +++ b/tests/run-with-compiler/staged-streams_1.scala @@ -78,6 +78,43 @@ object Test { // } } } + + + def map[B : Type](f: (Expr[A] => Expr[B])): Stream[B] = { + Stream(mapRaw[Expr[A], Expr[B]](a => k => '{ ~k(f(a)) }, stream)) + } + + def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), s: StagedStream[A]): StagedStream[B] = { + s match { + case Linear(producer) => { + val prod = new Producer[B] { + + type St = producer.St + + val card = producer.card + + def init(k: St => Expr[Unit]): Expr[Unit] = { + producer.init(k) + } + + def step(st: St, k: (B => Expr[Unit])): Expr[Unit] = { + producer.step(st, el => f(el)(k)) + } + + def hasNext(st: St): Expr[Boolean] = { + producer.hasNext(st) + } + } + + Linear(prod) + } + case nested: Nested[a, bt] => ??? +// { +// Nested(nested.producer, (a: bt) => mapRaw[A, B](f, nested.nestedf(a))) +// } + } + } + } object Stream { @@ -120,9 +157,15 @@ object Test { .of('{Array(1, 2, 3)}) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test2() = Stream + .of('{Array(1, 2, 3)}) + .map((a: Expr[Int]) => '{ ~a * 2 }) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) def main(args: Array[String]): Unit = { println(test1().run) + println + println(test2().run) } } From 58e172681f48e716da3f5f0d922d1d27466d6380 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 11 Apr 2018 16:00:11 +0200 Subject: [PATCH 07/28] Implement flatMap --- .../run-with-compiler/staged-streams_1.check | 4 ++- .../run-with-compiler/staged-streams_1.scala | 32 ++++++++++++++----- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/tests/run-with-compiler/staged-streams_1.check b/tests/run-with-compiler/staged-streams_1.check index 71e62be89621..044cc3b47ce3 100644 --- a/tests/run-with-compiler/staged-streams_1.check +++ b/tests/run-with-compiler/staged-streams_1.check @@ -1,3 +1,5 @@ 6 -12 \ No newline at end of file +12 + +36 \ No newline at end of file diff --git a/tests/run-with-compiler/staged-streams_1.scala b/tests/run-with-compiler/staged-streams_1.scala index 3d20c90cfac3..f90a3c7dc718 100644 --- a/tests/run-with-compiler/staged-streams_1.scala +++ b/tests/run-with-compiler/staged-streams_1.scala @@ -72,10 +72,9 @@ object Test { }) } } - case nested: Nested[a, bt] => ??? -// { -// fold_raw[bt](((e: bt) => fold_raw(consumer, nested.nestedf(e))), Linear(nested.producer)) -// } + case nested: Nested[a, bt] => { + fold_raw[bt](((e: bt) => fold_raw(consumer, nested.nestedf(e))), Linear(nested.producer)) + } } } @@ -108,13 +107,23 @@ object Test { Linear(prod) } - case nested: Nested[a, bt] => ??? -// { -// Nested(nested.producer, (a: bt) => mapRaw[A, B](f, nested.nestedf(a))) -// } + case nested: Nested[a, bt] => { + Nested(nested.producer, (a: bt) => mapRaw[A, B](f, nested.nestedf(a))) + } } } + def flatMap[B : Type](f: (Expr[A] => Stream[B])): Stream[B] = { + Stream(flatMapRaw[Expr[A], Expr[B]]((a => { val Stream (nested) = f(a); nested }), stream)) + } + + def flatMapRaw[A, B](f: (A => StagedStream[B]), stream: StagedStream[A]): StagedStream[B] = { + stream match { + case Linear(producer) => Nested(producer, f) + case nested: Nested[a, bt] => + Nested(nested.producer, (a: bt) => flatMapRaw[A, B](f, nested.nestedf(a))) + } + } } object Stream { @@ -162,10 +171,17 @@ object Test { .map((a: Expr[Int]) => '{ ~a * 2 }) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test3() = Stream + .of('{Array(1, 2, 3)}) + .flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d * ~dp })) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println println(test2().run) + println + println(test3().run) } } From 83f8ed247b71ef9710b68851d2de1580430af1d0 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 11 Apr 2018 17:17:54 +0200 Subject: [PATCH 08/28] Implement filter --- .../run-with-compiler/staged-streams_1.check | 4 ++- .../run-with-compiler/staged-streams_1.scala | 27 ++++++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/run-with-compiler/staged-streams_1.check b/tests/run-with-compiler/staged-streams_1.check index 044cc3b47ce3..c55469fb4c09 100644 --- a/tests/run-with-compiler/staged-streams_1.check +++ b/tests/run-with-compiler/staged-streams_1.check @@ -2,4 +2,6 @@ 12 -36 \ No newline at end of file +36 + +2 \ No newline at end of file diff --git a/tests/run-with-compiler/staged-streams_1.scala b/tests/run-with-compiler/staged-streams_1.scala index f90a3c7dc718..5373e615abf4 100644 --- a/tests/run-with-compiler/staged-streams_1.scala +++ b/tests/run-with-compiler/staged-streams_1.scala @@ -78,7 +78,6 @@ object Test { } } - def map[B : Type](f: (Expr[A] => Expr[B])): Stream[B] = { Stream(mapRaw[Expr[A], Expr[B]](a => k => '{ ~k(f(a)) }, stream)) } @@ -124,6 +123,25 @@ object Test { Nested(nested.producer, (a: bt) => flatMapRaw[A, B](f, nested.nestedf(a))) } } + + def filter(f: (Expr[A] => Expr[Boolean])): Stream[A] = { + val filterStream = (a: Expr[A]) => + new Producer[Expr[A]] { + type St = Expr[A] + val card = AtMost1 + + def init(k: St => Expr[Unit]): Expr[Unit] = + k(a) + + def step(st: St, k: (Expr[A] => Expr[Unit])): Expr[Unit] = + k(st) + + def hasNext(st: St): Expr[Boolean] = + f(st) + } + + Stream(flatMapRaw[Expr[A], Expr[A]]((a => { Linear(filterStream(a)) }), stream)) + } } object Stream { @@ -176,12 +194,19 @@ object Test { .flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d * ~dp })) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test4() = Stream + .of('{Array(1, 2, 3)}) + .filter((d: Expr[Int]) => '{ ~d % 2 == 0 }) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println println(test2().run) println println(test3().run) + println + println(test4().run) } } From 7bfbb06c87712040c8025f18ab3d3db2c75f2793 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 11 Apr 2018 19:35:29 +0200 Subject: [PATCH 09/28] Comment out implementation of take --- .../run-with-compiler/staged-streams_1.scala | 79 ++++++++++++++++++- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/tests/run-with-compiler/staged-streams_1.scala b/tests/run-with-compiler/staged-streams_1.scala index 5373e615abf4..fbfb1d5dbe4c 100644 --- a/tests/run-with-compiler/staged-streams_1.scala +++ b/tests/run-with-compiler/staged-streams_1.scala @@ -54,7 +54,7 @@ object Test { } } - def fold_raw[A](consumer: A => Expr[Unit], s: StagedStream[A]): Expr[Unit] = { + private def fold_raw[A](consumer: A => Expr[Unit], s: StagedStream[A]): Expr[Unit] = { s match { case Linear(producer) => { producer.card match { @@ -82,7 +82,7 @@ object Test { Stream(mapRaw[Expr[A], Expr[B]](a => k => '{ ~k(f(a)) }, stream)) } - def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), s: StagedStream[A]): StagedStream[B] = { + private def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), s: StagedStream[A]): StagedStream[B] = { s match { case Linear(producer) => { val prod = new Producer[B] { @@ -116,7 +116,7 @@ object Test { Stream(flatMapRaw[Expr[A], Expr[B]]((a => { val Stream (nested) = f(a); nested }), stream)) } - def flatMapRaw[A, B](f: (A => StagedStream[B]), stream: StagedStream[A]): StagedStream[B] = { + private def flatMapRaw[A, B](f: (A => StagedStream[B]), stream: StagedStream[A]): StagedStream[B] = { stream match { case Linear(producer) => Nested(producer, f) case nested: Nested[a, bt] => @@ -142,6 +142,79 @@ object Test { Stream(flatMapRaw[Expr[A], Expr[A]]((a => { Linear(filterStream(a)) }), stream)) } + + // def moreTermination[A](f: Rep[Boolean] => Rep[Boolean], stream: StagedStream[A]): StagedStream[A] = { + // def addToProducer[A](f: Rep[Boolean] => Rep[Boolean], producer: Producer[A]): Producer[A] = { + // producer.card match { + // case Many => + // new Producer[A] { + // type St = producer.St + + // val card = producer.card + // def init(k: St => Rep[Unit]): Rep[Unit] = + // producer.init(k) + // def step(st: St, k: (A => Rep[Unit])): Rep[Unit] = + // producer.step(st, el => k(el)) + // def hasNext(st: St): Rep[Boolean] = + // f(producer.hasNext(st)) + // } + // case AtMost1 => producer + // } + // } + // stream match { + // case Linear(producer) => Linear(addToProducer(f, producer)) + // case Nested(producer, nestedf) => + // Nested(addToProducer(f, producer), (a: Id[_]) => moreTermination(f, nestedf(a))) + // } + // } + + // private def addCounter[A](n: Expr[Int], producer: Producer[A]): Producer[(Expr[Int], A)] = + // new Producer[(Var[Int], A)] { + // type St = (Var[Int], producer.St) + + // val card = producer.card + // def init(k: St => Rep[Unit]): Rep[Unit] = { + // producer.init(st => { + // var counter: Var[Int] = n + // k(counter, st) + // }) + // } + // def step(st: St, k: (((Var[Int], A)) => Rep[Unit])): Rep[Unit] = { + // val (counter, nst) = st + // producer.step(nst, el => { + // k((counter, el)) + // }) + // } + // def hasNext(st: St): Rep[Boolean] = { + // val (counter, nst) = st + // producer.card match { + // case Many => counter > 0 && producer.hasNext(nst) + // case AtMost1 => producer.hasNext(nst) + // } + // } + // } + + // def takeRaw[A](n: Rep[Int], stream: StagedStream[A]): StagedStream[A] = { + // stream match { + // case Linear(producer) => { + // mapRaw[(Var[Int], A), A]((t => k => { + // t._1 = t._1 - 1 + // k(t._2) + // }), Linear(addCounter(n, producer))) + // } + // case Nested(producer, nestedf) => { + // Nested(addCounter(n, producer), (t: (Var[Int], Id[_])) => { + // mapRaw[A, A]((el => k => { + // t._1 = t._1 - 1 + // k(el) + // }), moreTermination(b => t._1 > 0 && b, nestedf(t._2))) + // }) + // } + // } + // } + + // def take(n: Rep[Int]): Stream[A] = Stream(takeRaw(n, stream)) + } object Stream { From ad06ab0834d29eeb277cb1baf9ccbae3c35cc0e1 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Fri, 13 Apr 2018 15:43:50 +0200 Subject: [PATCH 10/28] Implement take --- .../run-with-compiler/staged-streams_1.check | 6 +- .../run-with-compiler/staged-streams_1.scala | 164 ++++++++++-------- 2 files changed, 97 insertions(+), 73 deletions(-) diff --git a/tests/run-with-compiler/staged-streams_1.check b/tests/run-with-compiler/staged-streams_1.check index c55469fb4c09..31ba9aba99ad 100644 --- a/tests/run-with-compiler/staged-streams_1.check +++ b/tests/run-with-compiler/staged-streams_1.check @@ -4,4 +4,8 @@ 36 -2 \ No newline at end of file +2 + +3 + +7 \ No newline at end of file diff --git a/tests/run-with-compiler/staged-streams_1.scala b/tests/run-with-compiler/staged-streams_1.scala index fbfb1d5dbe4c..a8c946251a8e 100644 --- a/tests/run-with-compiler/staged-streams_1.scala +++ b/tests/run-with-compiler/staged-streams_1.scala @@ -88,7 +88,6 @@ object Test { val prod = new Producer[B] { type St = producer.St - val card = producer.card def init(k: St => Expr[Unit]): Expr[Unit] = { @@ -143,78 +142,84 @@ object Test { Stream(flatMapRaw[Expr[A], Expr[A]]((a => { Linear(filterStream(a)) }), stream)) } - // def moreTermination[A](f: Rep[Boolean] => Rep[Boolean], stream: StagedStream[A]): StagedStream[A] = { - // def addToProducer[A](f: Rep[Boolean] => Rep[Boolean], producer: Producer[A]): Producer[A] = { - // producer.card match { - // case Many => - // new Producer[A] { - // type St = producer.St - - // val card = producer.card - // def init(k: St => Rep[Unit]): Rep[Unit] = - // producer.init(k) - // def step(st: St, k: (A => Rep[Unit])): Rep[Unit] = - // producer.step(st, el => k(el)) - // def hasNext(st: St): Rep[Boolean] = - // f(producer.hasNext(st)) - // } - // case AtMost1 => producer - // } - // } - // stream match { - // case Linear(producer) => Linear(addToProducer(f, producer)) - // case Nested(producer, nestedf) => - // Nested(addToProducer(f, producer), (a: Id[_]) => moreTermination(f, nestedf(a))) - // } - // } - - // private def addCounter[A](n: Expr[Int], producer: Producer[A]): Producer[(Expr[Int], A)] = - // new Producer[(Var[Int], A)] { - // type St = (Var[Int], producer.St) - - // val card = producer.card - // def init(k: St => Rep[Unit]): Rep[Unit] = { - // producer.init(st => { - // var counter: Var[Int] = n - // k(counter, st) - // }) - // } - // def step(st: St, k: (((Var[Int], A)) => Rep[Unit])): Rep[Unit] = { - // val (counter, nst) = st - // producer.step(nst, el => { - // k((counter, el)) - // }) - // } - // def hasNext(st: St): Rep[Boolean] = { - // val (counter, nst) = st - // producer.card match { - // case Many => counter > 0 && producer.hasNext(nst) - // case AtMost1 => producer.hasNext(nst) - // } - // } - // } - - // def takeRaw[A](n: Rep[Int], stream: StagedStream[A]): StagedStream[A] = { - // stream match { - // case Linear(producer) => { - // mapRaw[(Var[Int], A), A]((t => k => { - // t._1 = t._1 - 1 - // k(t._2) - // }), Linear(addCounter(n, producer))) - // } - // case Nested(producer, nestedf) => { - // Nested(addCounter(n, producer), (t: (Var[Int], Id[_])) => { - // mapRaw[A, A]((el => k => { - // t._1 = t._1 - 1 - // k(el) - // }), moreTermination(b => t._1 > 0 && b, nestedf(t._2))) - // }) - // } - // } - // } - - // def take(n: Rep[Int]): Stream[A] = Stream(takeRaw(n, stream)) + private def moreTermination[A](f: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { + def addToProducer[A](f: Expr[Boolean] => Expr[Boolean], producer: Producer[A]): Producer[A] = { + producer.card match { + case Many => + new Producer[A] { + type St = producer.St + val card = producer.card + + def init(k: St => Expr[Unit]): Expr[Unit] = + producer.init(k) + + def step(st: St, k: (A => Expr[Unit])): Expr[Unit] = + producer.step(st, el => k(el)) + + def hasNext(st: St): Expr[Boolean] = + f(producer.hasNext(st)) + } + case AtMost1 => producer + } + } + + stream match { + case Linear(producer) => Linear(addToProducer(f, producer)) + case nested: Nested[a, bt] => + Nested(addToProducer(f, nested.producer), (a: bt) => moreTermination(f, nested.nestedf(a))) + } + } + + private def addCounter[A](n: Expr[Int], producer: Producer[A]): Producer[(Var[Int], A)] = { + new Producer[(Var[Int], A)] { + type St = (Var[Int], producer.St) + val card = producer.card + + def init(k: St => Expr[Unit]): Expr[Unit] = { + producer.init(st => { + Var(n) { counter => + k(counter, st) + } + }) + } + + def step(st: St, k: (((Var[Int], A)) => Expr[Unit])): Expr[Unit] = { + val (counter, nst) = st + producer.step(nst, el => '{ + ~k((counter, el)) + }) + } + + def hasNext(st: St): Expr[Boolean] = { + val (counter, nst) = st + producer.card match { + case Many => '{ ~counter.get > 0 && ~producer.hasNext(nst) } + case AtMost1 => '{ ~producer.hasNext(nst) } + } + } + } + } + + def takeRaw[A](n: Expr[Int], stream: StagedStream[A]): StagedStream[A] = { + stream match { + case Linear(producer) => { + mapRaw[(Var[Int], A), A]((t: (Var[Int], A)) => k => '{ + ~t._1.update('{~t._1.get - 1}) + ~k(t._2) + }, Linear(addCounter(n, producer))) + } + case nested: Nested[a, bt] => { + Nested(addCounter(n, nested.producer), (t: (Var[Int], bt)) => { + mapRaw[A, A]((el => k => '{ + ~t._1.update('{~t._1.get - 1}) + ~k(el) + }), moreTermination(b => '{ ~t._1.get > 0 && ~b}, nested.nestedf(t._2))) + }) + } + } + } + def take(n: Expr[Int]): Stream[A] = Stream(takeRaw[Expr[A]](n, stream)) } object Stream { @@ -272,6 +277,17 @@ object Test { .filter((d: Expr[Int]) => '{ ~d % 2 == 0 }) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test5() = Stream + .of('{Array(1, 2, 3)}) + .take('{2}) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + + def test6() = Stream + .of('{Array(1, 1, 1)}) + .flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).take('{2})) + .take('{5}) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println @@ -280,6 +296,10 @@ object Test { println(test3().run) println println(test4().run) + println + println(test5().run) + println + println(test6().run) } } From a2e4c495a996ee6d1f81a174cc5ce31e917408c4 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Fri, 13 Apr 2018 17:59:34 +0200 Subject: [PATCH 11/28] Move to custom args for run-with-compiler --- compiler/test/dotty/tools/dotc/CompilationTests.scala | 3 ++- .../staged-streams_1.check | 0 .../staged-streams_1.scala | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) rename tests/{run-with-compiler => run-with-compiler-custom-args}/staged-streams_1.check (100%) rename tests/{run-with-compiler => run-with-compiler-custom-args}/staged-streams_1.scala (98%) diff --git a/compiler/test/dotty/tools/dotc/CompilationTests.scala b/compiler/test/dotty/tools/dotc/CompilationTests.scala index aaa8ac216b5b..73915721639a 100644 --- a/compiler/test/dotty/tools/dotc/CompilationTests.scala +++ b/compiler/test/dotty/tools/dotc/CompilationTests.scala @@ -198,7 +198,8 @@ class CompilationTests extends ParallelTesting { implicit val testGroup: TestGroup = TestGroup("runAll") compileFilesInDir("tests/run", defaultOptions) + compileFilesInDir("tests/run-no-optimise", defaultOptions) + - compileFilesInDir("tests/run-with-compiler", defaultRunWithCompilerOptions) + compileFilesInDir("tests/run-with-compiler", defaultRunWithCompilerOptions) + + compileFile("tests/run-with-compiler-custom-args/staged-streams_1.scala", defaultRunWithCompilerOptions without "-Yno-deep-subtypes") }.checkRuns() // Generic java signatures tests --------------------------------------------- diff --git a/tests/run-with-compiler/staged-streams_1.check b/tests/run-with-compiler-custom-args/staged-streams_1.check similarity index 100% rename from tests/run-with-compiler/staged-streams_1.check rename to tests/run-with-compiler-custom-args/staged-streams_1.check diff --git a/tests/run-with-compiler/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala similarity index 98% rename from tests/run-with-compiler/staged-streams_1.scala rename to tests/run-with-compiler-custom-args/staged-streams_1.scala index a8c946251a8e..435882aeffce 100644 --- a/tests/run-with-compiler/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -73,7 +73,7 @@ object Test { } } case nested: Nested[a, bt] => { - fold_raw[bt](((e: bt) => fold_raw(consumer, nested.nestedf(e))), Linear(nested.producer)) + fold_raw[bt](((e: bt) => fold_raw[a](consumer, nested.nestedf(e))), Linear(nested.producer)) } } } From 0328028f422754525820ca9a2db4fa9b87042612 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 17 Apr 2018 16:22:25 +0200 Subject: [PATCH 12/28] Implement zip (for linear streams) --- .../staged-streams_1.check | 4 +- .../staged-streams_1.scala | 58 +++++++++++++++++-- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.check b/tests/run-with-compiler-custom-args/staged-streams_1.check index 31ba9aba99ad..3158b3eb5f84 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.check +++ b/tests/run-with-compiler-custom-args/staged-streams_1.check @@ -8,4 +8,6 @@ 3 -7 \ No newline at end of file +7 + +12 \ No newline at end of file diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 435882aeffce..5eed512595c2 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -82,8 +82,8 @@ object Test { Stream(mapRaw[Expr[A], Expr[B]](a => k => '{ ~k(f(a)) }, stream)) } - private def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), s: StagedStream[A]): StagedStream[B] = { - s match { + private def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), stream: StagedStream[A]): StagedStream[B] = { + stream match { case Linear(producer) => { val prod = new Producer[B] { @@ -200,7 +200,7 @@ object Test { } } - def takeRaw[A](n: Expr[Int], stream: StagedStream[A]): StagedStream[A] = { + private def takeRaw[A](n: Expr[Int], stream: StagedStream[A]): StagedStream[A] = { stream match { case Linear(producer) => { mapRaw[(Var[Int], A), A]((t: (Var[Int], A)) => k => '{ @@ -219,7 +219,50 @@ object Test { } } - def take(n: Expr[Int]): Stream[A] = Stream(takeRaw[Expr[A]](n, stream)) + def take(n: Expr[Int]): Stream[A] = Stream(takeRaw[Expr[A]](n, stream)) + + private def zipRaw[A, B](stream1: StagedStream[A], stream2: StagedStream[B]): StagedStream[(A, B)] = { + (stream1, stream2) match { + + case (Linear(producer1), Linear(producer2)) => + Linear(zip_producer(producer1, producer2)) + + case (Linear(producer1), Nested(producer2, nestf2)) => ??? + + case (Nested(producer1, nestf1), Linear(producer2)) => ??? + + case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => ??? + } + } + + private def zip_producer[A, B](producer1: Producer[A], producer2: Producer[B]) = { + new Producer[(A, B)] { + type St = (producer1.St, producer2.St) + + val card: Cardinality = Many + + def init(k: St => Expr[Unit]): Expr[Unit] = { + producer1.init(s1 => '{ ~producer2.init(s2 => '{ ~k((s1, s2)) })}) + } + + def step(st: St, k: ((A, B)) => Expr[Unit]): Expr[Unit] = { + val (s1, s2) = st + producer1.step(s1, el1 => '{ ~producer2.step(s2, el2 => '{ ~k((el1, el2)) })}) + } + + def hasNext(st: St): Expr[Boolean] = { + val (s1, s2) = st + '{ ~producer1.hasNext(s1) && ~producer2.hasNext(s2) } + } + } + } + + def zip[B : Type, C : Type](f: (Expr[A] => Expr[B] => Expr[C]), stream2: Stream[B]): Stream[C] = { + + val Stream(stream_b) = stream2 + + Stream(mapRaw[(Expr[A], Expr[B]), Expr[C]]((t => k => '{ ~k(f(t._1)(t._2)) }), zipRaw[Expr[A], Expr[B]](stream, stream_b))) + } } object Stream { @@ -288,6 +331,11 @@ object Test { .take('{5}) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test7() = Stream + .of('{Array(1, 2, 3)}) + .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)})) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println @@ -300,6 +348,8 @@ object Test { println(test5().run) println println(test6().run) + println + println(test7().run) } } From 9748f0daf46bbc6dade371f0463bd3a7e0de534d Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 17 Apr 2018 17:24:17 +0200 Subject: [PATCH 13/28] Implement zip (linear/nested) --- .../staged-streams_1.check | 4 +- .../staged-streams_1.scala | 46 ++++++++++++++++++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.check b/tests/run-with-compiler-custom-args/staged-streams_1.check index 3158b3eb5f84..6ecaa75d3905 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.check +++ b/tests/run-with-compiler-custom-args/staged-streams_1.check @@ -10,4 +10,6 @@ 7 -12 \ No newline at end of file +12 + +15 \ No newline at end of file diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 5eed512595c2..3adf58bb8f1a 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -227,7 +227,8 @@ object Test { case (Linear(producer1), Linear(producer2)) => Linear(zip_producer(producer1, producer2)) - case (Linear(producer1), Nested(producer2, nestf2)) => ??? + case (Linear(producer1), Nested(producer2, nestf2)) => + pushLinear(producer1, producer2, nestf2) case (Nested(producer1, nestf1), Linear(producer2)) => ??? @@ -235,10 +236,44 @@ object Test { } } + private def pushLinear[A, B, C](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { + val newProducer = new Producer[(Var[Boolean], producer.St, B)] { + + type St = (Var[Boolean], producer.St, nestedProducer.St) + val card: Cardinality = Many + + def init(k: St => Expr[Unit]): Expr[Unit] = { + producer.init(s1 => '{ ~nestedProducer.init(s2 => + Var('{ ~producer.hasNext(s1) }) { term1r => + k((term1r, s1, s2)) + })}) + } + + def step(st: St, k: ((Var[Boolean], producer.St, B)) => Expr[Unit]): Expr[Unit] = { + val (flag, s1, s2) = st + nestedProducer.step(s2, b => '{ ~k((flag, s1, b)) }) + } + + def hasNext(st: St): Expr[Boolean] = { + val (flag, s1, s2) = st + '{ ~flag.get && ~nestedProducer.hasNext(s2) } + } + } + + Nested(newProducer, (t: (Var[Boolean], producer.St, B)) => { + val (flag, s1, b) = t + + mapRaw[C, (A, C)]((c => k => '{ + ~producer.step(s1, a => '{ ~k((a, c)) }) + ~flag.update(producer.hasNext(s1)) + }), moreTermination((b_flag: Expr[Boolean]) => '{ ~flag.get && ~b_flag }, nestedf(b))) + }) + } + private def zip_producer[A, B](producer1: Producer[A], producer2: Producer[B]) = { new Producer[(A, B)] { - type St = (producer1.St, producer2.St) + type St = (producer1.St, producer2.St) val card: Cardinality = Many def init(k: St => Expr[Unit]): Expr[Unit] = { @@ -336,6 +371,11 @@ object Test { .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)})) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test8() = Stream + .of('{Array(1, 2, 3)}) + .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)}).flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d + ~dp }))) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println @@ -350,6 +390,8 @@ object Test { println(test6().run) println println(test7().run) + println + println(test8().run) } } From 6dfbe9d83046a6119f8cc2add823be6cf9e99887 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 17 Apr 2018 18:07:08 +0200 Subject: [PATCH 14/28] Implement zip (nested/linear) --- .../staged-streams_1.check | 2 ++ .../staged-streams_1.scala | 12 ++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.check b/tests/run-with-compiler-custom-args/staged-streams_1.check index 6ecaa75d3905..a9891d75b82a 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.check +++ b/tests/run-with-compiler-custom-args/staged-streams_1.check @@ -12,4 +12,6 @@ 12 +15 + 15 \ No newline at end of file diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 3adf58bb8f1a..cafd736b8dbb 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -228,9 +228,10 @@ object Test { Linear(zip_producer(producer1, producer2)) case (Linear(producer1), Nested(producer2, nestf2)) => - pushLinear(producer1, producer2, nestf2) + pushLinear[A, _, B](producer1, producer2, nestf2) - case (Nested(producer1, nestf1), Linear(producer2)) => ??? + case (Nested(producer1, nestf1), Linear(producer2)) => + mapRaw[(B, A), (A, B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, A](producer2, producer1, nestf1)) case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => ??? } @@ -376,6 +377,11 @@ object Test { .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)}).flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d + ~dp }))) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test9() = Stream + .of('{Array(1, 2, 3)}).flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d + ~dp })) + .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)}) ) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println @@ -392,6 +398,8 @@ object Test { println(test7().run) println println(test8().run) + println + println(test9().run) } } From bdfdb75a379ba20393e65811bbba18fa83b0307a Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 18 Apr 2018 17:09:03 +0200 Subject: [PATCH 15/28] Refactor and add comments --- .../staged-streams_1.scala | 130 ++++++++++++++---- 1 file changed, 106 insertions(+), 24 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index cafd736b8dbb..d212149ef0e1 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -44,9 +44,8 @@ object Test { def fold[W: Type](z: Expr[W], f: ((Expr[W], Expr[A]) => Expr[W])): Expr[W] = { Var(z) { s: Var[W] => '{ - ~fold_raw[Expr[A]]((a: Expr[A]) => '{ - ~s.update(f(s.get, a)) + ~s.update(f(s.get, a)) }, stream) ~s.get @@ -54,8 +53,8 @@ object Test { } } - private def fold_raw[A](consumer: A => Expr[Unit], s: StagedStream[A]): Expr[Unit] = { - s match { + private def fold_raw[A](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { + stream match { case Linear(producer) => { producer.card match { case Many => @@ -72,16 +71,36 @@ object Test { }) } } - case nested: Nested[a, bt] => { - fold_raw[bt](((e: bt) => fold_raw[a](consumer, nested.nestedf(e))), Linear(nested.producer)) + case nested: Nested[A, bt] => { + fold_raw[bt](((e: bt) => fold_raw[A](consumer, nested.nestedf(e))), Linear(nested.producer)) } } } + /** Builds a new stream by applying a function to all elements of this stream. + * + * @param f the function to apply to each quoted element. + * @tparam B the element type of the returned stream + * @return a new stream resulting from applying `mapRaw` and threading the element of the first stream downstream. + */ def map[B : Type](f: (Expr[A] => Expr[B])): Stream[B] = { Stream(mapRaw[Expr[A], Expr[B]](a => k => '{ ~k(f(a)) }, stream)) } + /** Handles generically the mapping of elements from one producer to another. + * `mapRaw` can be potentially used threading quoted values from one stream to another. However + * is can be also used by handling any kind of quoted value. + * + * e.g., `mapRaw[(Var[Int], A), A]` transforms a stream that declares a variable and holds a value in each + * iteration step to a stream that is not aware of the aforementioned variable. + * + * @param f the function to apply at each step. f is of type `(A => (B => Expr[Unit])` where A is the type of + * the incoming stream. When applied to an element, `f` returns the continuation for elements of `B` + * @param stream that contains the stream we want to map. + * @tparam A the type of the input stream + * @tparam B the element type of the resulting stream + * @return a new stream resulting from applying `f` in the `step` function of the input stream's producer. + */ private def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), stream: StagedStream[A]): StagedStream[B] = { stream match { case Linear(producer) => { @@ -105,16 +124,30 @@ object Test { Linear(prod) } - case nested: Nested[a, bt] => { + case nested: Nested[A, bt] => { Nested(nested.producer, (a: bt) => mapRaw[A, B](f, nested.nestedf(a))) } } } + /** Flatmap */ def flatMap[B : Type](f: (Expr[A] => Stream[B])): Stream[B] = { Stream(flatMapRaw[Expr[A], Expr[B]]((a => { val Stream (nested) = f(a); nested }), stream)) } + /** Returns a new stream that applies a function `f` to each element of the input stream. + * If the input stream is simply linear then its packed with the function `f`. + * If the input stream is nested then a new one is created by using its producer and then passing the `f` + * recursively to build the `nestedf` of the returned stream. + * + * Note: always returns a nested stream. + * + * @param f the function of `flatMap`` + * @param stream the input stream + * @tparam A the type of the input stream + * @tparam B the element type of the resulting stream + * @return a new stream resulting from registering `f` + */ private def flatMapRaw[A, B](f: (A => StagedStream[B]), stream: StagedStream[A]): StagedStream[B] = { stream match { case Linear(producer) => Nested(producer, f) @@ -123,9 +156,19 @@ object Test { } } - def filter(f: (Expr[A] => Expr[Boolean])): Stream[A] = { + /** Selects all elements of this stream which satisfy a predicate. + * + * Note: this is merely a special case of `flatMap` as the resulting stream in each step may return 0 or 1 + * element. + * + * @param f the predicate used to test elements. + * @return a new stream consisting of all elements of the input stream that do satisfy the given + * predicate `pred`. + */ + def filter(pred: (Expr[A] => Expr[Boolean])): Stream[A] = { val filterStream = (a: Expr[A]) => new Producer[Expr[A]] { + type St = Expr[A] val card = AtMost1 @@ -136,13 +179,22 @@ object Test { k(st) def hasNext(st: St): Expr[Boolean] = - f(st) + pred(st) } Stream(flatMapRaw[Expr[A], Expr[A]]((a => { Linear(filterStream(a)) }), stream)) } - private def moreTermination[A](f: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { + /** Adds a new termination condition to a producer of cardinality `Many`. + * + * @param condition the termination condition as a function accepting the existing condition (the result + * of the `hasNext` from the passed `stream`'s producer. + * @param stream that contains the producer we want to enhance. + * @tparam A the type of the stream's elements. + * @return the stream with the new producer. If the passed stream was linear, the new termination is added + * otherwise the new termination is propagated to all nested ones, recursively. + */ + private def addTerminationCondition[A](condition: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { def addToProducer[A](f: Expr[Boolean] => Expr[Boolean], producer: Producer[A]): Producer[A] = { producer.card match { case Many => @@ -164,12 +216,20 @@ object Test { } stream match { - case Linear(producer) => Linear(addToProducer(f, producer)) + case Linear(producer) => Linear(addToProducer(condition, producer)) case nested: Nested[a, bt] => - Nested(addToProducer(f, nested.producer), (a: bt) => moreTermination(f, nested.nestedf(a))) + Nested(addToProducer(condition, nested.producer), (a: bt) => addTerminationCondition(condition, nested.nestedf(a))) } } + /** Adds a new counter variable by enhancing a producer's state with a variable of type `Int`. + * The counter is initialized in `init`, propageted in `step` and checked in the `hasNext` of the *current* stream. + * + * @param n is the initial value of the counter + * @param producer the producer that we want to enhance + * @tparam A the type of the producer's elements. + * @return the enhanced producer + */ private def addCounter[A](n: Expr[Int], producer: Producer[A]): Producer[(Var[Int], A)] = { new Producer[(Var[Int], A)] { type St = (Var[Int], producer.St) @@ -184,41 +244,58 @@ object Test { } def step(st: St, k: (((Var[Int], A)) => Expr[Unit])): Expr[Unit] = { - val (counter, nst) = st - producer.step(nst, el => '{ + val (counter, currentState) = st + producer.step(currentState, el => '{ ~k((counter, el)) }) } def hasNext(st: St): Expr[Boolean] = { - val (counter, nst) = st + val (counter, currentState) = st producer.card match { - case Many => '{ ~counter.get > 0 && ~producer.hasNext(nst) } - case AtMost1 => '{ ~producer.hasNext(nst) } + case Many => '{ ~counter.get > 0 && ~producer.hasNext(currentState) } + case AtMost1 => '{ ~producer.hasNext(currentState) } } } } } + /** The nested stream receives the same variable reference; thus all streams decrement the same global count. + * + * @param n code of the variable to be threaded to the downstream. + * @param stream the upstream to enhance. + * @tparam A the type of the producer's elements. + * @return a linear or nested stream aware of the variable reference to decrement. + */ private def takeRaw[A](n: Expr[Int], stream: StagedStream[A]): StagedStream[A] = { stream match { - case Linear(producer) => { + case linear: Linear[A] => { + val enhancedProducer: Producer[(Var[Int], A)] = addCounter[A](n, linear.producer) + val enhancedStream: Linear[(Var[Int], A)] = Linear(enhancedProducer) + + // Map an enhanced stream to a stream that produces the elements. Before + // invoking the continuation for the element, "use" the variable accordingly. mapRaw[(Var[Int], A), A]((t: (Var[Int], A)) => k => '{ ~t._1.update('{~t._1.get - 1}) ~k(t._2) - }, Linear(addCounter(n, producer))) + }, enhancedStream) } - case nested: Nested[a, bt] => { - Nested(addCounter(n, nested.producer), (t: (Var[Int], bt)) => { + case nested: Nested[A, bt] => { + val enhancedProducer: Producer[(Var[Int], bt)] = addCounter[bt](n, nested.producer) + + Nested(enhancedProducer, (t: (Var[Int], bt)) => { + // Before invoking the continuation for the element, "use" the variable accordingly. + // In contrast to the linear case, the variable is initialized in the originating stream. mapRaw[A, A]((el => k => '{ ~t._1.update('{~t._1.get - 1}) ~k(el) - }), moreTermination(b => '{ ~t._1.get > 0 && ~b}, nested.nestedf(t._2))) + }), addTerminationCondition(b => '{ ~t._1.get > 0 && ~b}, nested.nestedf(t._2))) }) } } } + /** A stream containing the first `n` elements of this stream. */ def take(n: Expr[Int]): Stream[A] = Stream(takeRaw[Expr[A]](n, stream)) private def zipRaw[A, B](stream1: StagedStream[A], stream2: StagedStream[B]): StagedStream[(A, B)] = { @@ -233,10 +310,15 @@ object Test { case (Nested(producer1, nestf1), Linear(producer2)) => mapRaw[(B, A), (A, B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, A](producer2, producer1, nestf1)) - case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => ??? + case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => + zipRaw(makeLinear(stream1), stream2) } } + private def makeLinear[A](stream: StagedStream[A]): StagedStream[A] = { + ??? + } + private def pushLinear[A, B, C](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { val newProducer = new Producer[(Var[Boolean], producer.St, B)] { @@ -267,7 +349,7 @@ object Test { mapRaw[C, (A, C)]((c => k => '{ ~producer.step(s1, a => '{ ~k((a, c)) }) ~flag.update(producer.hasNext(s1)) - }), moreTermination((b_flag: Expr[Boolean]) => '{ ~flag.get && ~b_flag }, nestedf(b))) + }), addTerminationCondition((b_flag: Expr[Boolean]) => '{ ~flag.get && ~b_flag }, nestedf(b))) }) } From d565b1b3adc53f198e70cebefe487178425592cf Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Thu, 19 Apr 2018 17:26:44 +0200 Subject: [PATCH 16/28] WIP implemention of reifying nested streams for zip --- .../staged-streams_1.scala | 89 ++++++++++++++++++- 1 file changed, 88 insertions(+), 1 deletion(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index d212149ef0e1..069015aeb7ec 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -315,8 +315,95 @@ object Test { } } + /** + * + * @param stream + * @tparam A + * @return + */ private def makeLinear[A](stream: StagedStream[A]): StagedStream[A] = { - ??? + stream match { + case Linear(producer) => stream + case Nested(producer, nestedf) => { + + /** Helper function that orchestrates the handling of the function that represents an `advance: Unit => Unit`. + * It reifies a nested stream as calls to `advance`. Advance encodes the step function of each nested stream. + * It is used in the init of a producer of a nested stream. When an inner stream finishes, the + * `currentAdvance` holds the function to the `advance` function of the earlier stream. + * `makeAdvanceFunction`, for each nested stream, installs a new `advance` function that after + * the stream finishes it will restore the earlier one. + * + * When `advance` is called the result is consumed in the continuation. Within this continuation + * the resulting value should be saved in a variable. + * + * @param currentAdvance variable that holds a function that represents the stream at each level. + * @param k the continuation that consumes a variable. + * @return the quote of the orchestrated code that will be executed as + */ + def makeAdvanceFunction[A](currentAdvance: Var[Unit => Unit], k: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { + stream match { + case Linear(producer) => + producer.card match { + case AtMost1 => producer.init(st => '{ + if(~producer.hasNext(st)) { + ~producer.step(st, k) + } + }) + case Many => producer.init(st => '{ + val oldAdvance : Unit => Unit = ~currentAdvance.get + val newAdvance : Unit => Unit = { _: Unit => { + if(~producer.hasNext(st)) { + ~producer.step(st, k) + } + else { + ~currentAdvance.update('{oldAdvance}) + } + ~currentAdvance.update('{newAdvance}) + newAdvance(_) + }} + }) + } + case nested: Nested[A, bt] => + makeAdvanceFunction(currentAdvance, (a: bt) => makeAdvanceFunction(currentAdvance, k, nested.nestedf(a)), Linear(nested.producer)) + } + } + + val newProducer = new Producer[A] { + // _1: if the stream has ended, + // _2: the current element, + // _3: the step of the inner most steam + type St = (Var[Boolean], Var[A], Var[Unit => Unit]) + val card: Cardinality = Many + + def init(k: St => Expr[Unit]): Expr[Unit] = { + producer.init(st => '{ + Var('{ (_: Unit) => ()}){ advf => { + Var('{ true }) { hasNext => { + // Var('{ null.asInstanceOf[A] }) { curr => { + // def adv() = { + // ~hasNext.update(producer.hasNext(st)) + // if(~hasNext.get) { + // ~producer.step(st, el => '{ + // ~makeAdvanceFunction(advf, ((a: A) => curr.update('{a})), nestedf(el)) + // }) + // } + // } + // ~k((flag, current, advf)) + ??? + // }} + }} + }} + }) + } + + def step(st: St, k: A => Expr[Unit]): Expr[Unit] = ??? + + def hasNext(st: St): Expr[Boolean] = ??? + } + + Linear(newProducer) + } + } } private def pushLinear[A, B, C](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { From 01058cc53337822dd663b41c17bd3a650a7b391f Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 24 Apr 2018 11:00:25 +0200 Subject: [PATCH 17/28] Add Type annotations --- .../staged-streams_1.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 069015aeb7ec..3fadda68efb4 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -40,7 +40,7 @@ object Test { case class Linear[A](producer: Producer[A]) extends StagedStream[A] case class Nested[A, B](producer: Producer[B], nestedf: B => StagedStream[A]) extends StagedStream[A] - case class Stream[A](stream: StagedStream[Expr[A]]) { + case class Stream[A: Type](stream: StagedStream[Expr[A]]) { def fold[W: Type](z: Expr[W], f: ((Expr[W], Expr[A]) => Expr[W])): Expr[W] = { Var(z) { s: Var[W] => '{ @@ -101,7 +101,7 @@ object Test { * @tparam B the element type of the resulting stream * @return a new stream resulting from applying `f` in the `step` function of the input stream's producer. */ - private def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), stream: StagedStream[A]): StagedStream[B] = { + private def mapRaw[A: Type, B: Type](f: (A => (B => Expr[Unit]) => Expr[Unit]), stream: StagedStream[A]): StagedStream[B] = { stream match { case Linear(producer) => { val prod = new Producer[B] { @@ -194,7 +194,7 @@ object Test { * @return the stream with the new producer. If the passed stream was linear, the new termination is added * otherwise the new termination is propagated to all nested ones, recursively. */ - private def addTerminationCondition[A](condition: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { + private def addTerminationCondition[A: Type](condition: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { def addToProducer[A](f: Expr[Boolean] => Expr[Boolean], producer: Producer[A]): Producer[A] = { producer.card match { case Many => @@ -267,7 +267,7 @@ object Test { * @tparam A the type of the producer's elements. * @return a linear or nested stream aware of the variable reference to decrement. */ - private def takeRaw[A](n: Expr[Int], stream: StagedStream[A]): StagedStream[A] = { + private def takeRaw[A: Type](n: Expr[Int], stream: StagedStream[A]): StagedStream[A] = { stream match { case linear: Linear[A] => { val enhancedProducer: Producer[(Var[Int], A)] = addCounter[A](n, linear.producer) @@ -298,7 +298,7 @@ object Test { /** A stream containing the first `n` elements of this stream. */ def take(n: Expr[Int]): Stream[A] = Stream(takeRaw[Expr[A]](n, stream)) - private def zipRaw[A, B](stream1: StagedStream[A], stream2: StagedStream[B]): StagedStream[(A, B)] = { + private def zipRaw[A: Type, B: Type](stream1: StagedStream[A], stream2: StagedStream[B]): StagedStream[(A, B)] = { (stream1, stream2) match { case (Linear(producer1), Linear(producer2)) => @@ -321,7 +321,7 @@ object Test { * @tparam A * @return */ - private def makeLinear[A](stream: StagedStream[A]): StagedStream[A] = { + private def makeLinear[A: Type](stream: StagedStream[A]): StagedStream[A] = { stream match { case Linear(producer) => stream case Nested(producer, nestedf) => { @@ -406,7 +406,7 @@ object Test { } } - private def pushLinear[A, B, C](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { + private def pushLinear[A: Type, B, C: Type](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { val newProducer = new Producer[(Var[Boolean], producer.St, B)] { type St = (Var[Boolean], producer.St, nestedProducer.St) @@ -440,7 +440,7 @@ object Test { }) } - private def zip_producer[A, B](producer1: Producer[A], producer2: Producer[B]) = { + private def zip_producer[A: Type, B: Type](producer1: Producer[A], producer2: Producer[B]) = { new Producer[(A, B)] { type St = (producer1.St, producer2.St) From 395bd6d3b5eecd1ed3560c2ce933a86ebe85a721 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 24 Apr 2018 18:45:21 +0200 Subject: [PATCH 18/28] Progress in makeLinear --- .../staged-streams_1.scala | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 3fadda68efb4..e867538070af 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -311,7 +311,7 @@ object Test { mapRaw[(B, A), (A, B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, A](producer2, producer1, nestf1)) case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => - zipRaw(makeLinear(stream1), stream2) + zipRaw(Linear(makeLinear(stream1)), stream2) } } @@ -321,10 +321,12 @@ object Test { * @tparam A * @return */ - private def makeLinear[A: Type](stream: StagedStream[A]): StagedStream[A] = { + private def makeLinear[A: Type](stream: StagedStream[A]): Producer[A] = { stream match { - case Linear(producer) => stream - case Nested(producer, nestedf) => { + case Linear(producer) => producer + case nested: Nested[A, bt] => { + val producer: Producer[bt] = nested.producer + val nestedf: bt => StagedStream[A] = nested.nestedf /** Helper function that orchestrates the handling of the function that represents an `advance: Unit => Unit`. * It reifies a nested stream as calls to `advance`. Advance encodes the step function of each nested stream. @@ -368,7 +370,7 @@ object Test { } } - val newProducer = new Producer[A] { + new Producer[A] { // _1: if the stream has ended, // _2: the current element, // _3: the step of the inner most steam @@ -376,32 +378,31 @@ object Test { val card: Cardinality = Many def init(k: St => Expr[Unit]): Expr[Unit] = { - producer.init(st => '{ + producer.init(st => Var('{ (_: Unit) => ()}){ advf => { Var('{ true }) { hasNext => { - // Var('{ null.asInstanceOf[A] }) { curr => { - // def adv() = { - // ~hasNext.update(producer.hasNext(st)) - // if(~hasNext.get) { - // ~producer.step(st, el => '{ - // ~makeAdvanceFunction(advf, ((a: A) => curr.update('{a})), nestedf(el)) - // }) - // } - // } - // ~k((flag, current, advf)) - ??? - // }} + Var('{ null.asInstanceOf[A] }) { curr => '{ + + // val adv: Unit => Unit = { + // ~hasNext.update(producer.hasNext(st)) + // if(~hasNext.get) { + // ~producer.step(st, (el: bt) => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) + // } + // } + + // ~advf.update('{adv}) + // adv(_) + + ~k((hasNext, curr, advf)) + }} }} - }} - }) + }}) } def step(st: St, k: A => Expr[Unit]): Expr[Unit] = ??? def hasNext(st: St): Expr[Boolean] = ??? } - - Linear(newProducer) } } } From 4027fa260328766934e9e9e96db295ff106e5eca Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 25 Apr 2018 15:47:33 +0200 Subject: [PATCH 19/28] Fix types for zip --- .../staged-streams_1.scala | 33 +++++++++---------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index e867538070af..8b2b650fbc1b 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -298,20 +298,20 @@ object Test { /** A stream containing the first `n` elements of this stream. */ def take(n: Expr[Int]): Stream[A] = Stream(takeRaw[Expr[A]](n, stream)) - private def zipRaw[A: Type, B: Type](stream1: StagedStream[A], stream2: StagedStream[B]): StagedStream[(A, B)] = { + private def zipRaw[A: Type, B: Type](stream1: StagedStream[Expr[A]], stream2: StagedStream[B]): StagedStream[(Expr[A], B)] = { (stream1, stream2) match { case (Linear(producer1), Linear(producer2)) => Linear(zip_producer(producer1, producer2)) case (Linear(producer1), Nested(producer2, nestf2)) => - pushLinear[A, _, B](producer1, producer2, nestf2) + pushLinear[Expr[A], _, B](producer1, producer2, nestf2) case (Nested(producer1, nestf1), Linear(producer2)) => - mapRaw[(B, A), (A, B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, A](producer2, producer1, nestf1)) + mapRaw[(B, Expr[A]), (Expr[A], B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, Expr[A]](producer2, producer1, nestf1)) case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => - zipRaw(Linear(makeLinear(stream1)), stream2) + zipRaw(Linear(makeLinear[A](stream1)), stream2) } } @@ -321,7 +321,7 @@ object Test { * @tparam A * @return */ - private def makeLinear[A: Type](stream: StagedStream[A]): Producer[A] = { + private def makeLinear[A: Type](stream: StagedStream[Expr[A]]): Producer[Expr[A]] = { stream match { case Linear(producer) => producer case nested: Nested[A, bt] => { @@ -370,7 +370,7 @@ object Test { } } - new Producer[A] { + new Producer[Expr[A]] { // _1: if the stream has ended, // _2: the current element, // _3: the step of the inner most steam @@ -382,16 +382,15 @@ object Test { Var('{ (_: Unit) => ()}){ advf => { Var('{ true }) { hasNext => { Var('{ null.asInstanceOf[A] }) { curr => '{ + val adv: Unit => Unit = { _ => + ~hasNext.update(producer.hasNext(st)) + if(~hasNext.get) { + // ~producer.step(st, (el: bt) => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) + } + } - // val adv: Unit => Unit = { - // ~hasNext.update(producer.hasNext(st)) - // if(~hasNext.get) { - // ~producer.step(st, (el: bt) => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) - // } - // } - - // ~advf.update('{adv}) - // adv(_) + ~advf.update('{adv}) + adv(()) ~k((hasNext, curr, advf)) }} @@ -399,7 +398,7 @@ object Test { }}) } - def step(st: St, k: A => Expr[Unit]): Expr[Unit] = ??? + def step(st: St, k: Expr[A] => Expr[Unit]): Expr[Unit] = ??? def hasNext(st: St): Expr[Boolean] = ??? } @@ -467,7 +466,7 @@ object Test { val Stream(stream_b) = stream2 - Stream(mapRaw[(Expr[A], Expr[B]), Expr[C]]((t => k => '{ ~k(f(t._1)(t._2)) }), zipRaw[Expr[A], Expr[B]](stream, stream_b))) + Stream(mapRaw[(Expr[A], Expr[B]), Expr[C]]((t => k => '{ ~k(f(t._1)(t._2)) }), zipRaw[A, Expr[B]](stream, stream_b))) } } From d38fc836c29153724e79820afb229f1ed236b1af Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Thu, 26 Apr 2018 16:08:27 +0200 Subject: [PATCH 20/28] Compile OK --- tests/run-with-compiler-custom-args/staged-streams_1.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 8b2b650fbc1b..2ed27ed1974f 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -324,10 +324,7 @@ object Test { private def makeLinear[A: Type](stream: StagedStream[Expr[A]]): Producer[Expr[A]] = { stream match { case Linear(producer) => producer - case nested: Nested[A, bt] => { - val producer: Producer[bt] = nested.producer - val nestedf: bt => StagedStream[A] = nested.nestedf - + case Nested(producer, nestedf) => { /** Helper function that orchestrates the handling of the function that represents an `advance: Unit => Unit`. * It reifies a nested stream as calls to `advance`. Advance encodes the step function of each nested stream. * It is used in the init of a producer of a nested stream. When an inner stream finishes, the @@ -385,7 +382,7 @@ object Test { val adv: Unit => Unit = { _ => ~hasNext.update(producer.hasNext(st)) if(~hasNext.get) { - // ~producer.step(st, (el: bt) => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) + ~producer.step(st, el => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) } } From 0d02ceab477e14dd1f90562dc7bc2fdf492b4df9 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Fri, 27 Apr 2018 13:47:09 +0200 Subject: [PATCH 21/28] Rebase and adjust type signatures --- .../run-with-compiler-custom-args/staged-streams_1.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 2ed27ed1974f..76ceeeb16ccb 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -310,8 +310,8 @@ object Test { case (Nested(producer1, nestf1), Linear(producer2)) => mapRaw[(B, Expr[A]), (Expr[A], B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, Expr[A]](producer2, producer1, nestf1)) - case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => - zipRaw(Linear(makeLinear[A](stream1)), stream2) + case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => ??? + // zipRaw[A, B](Linear(makeLinear(stream1)), stream2) } } @@ -437,7 +437,7 @@ object Test { }) } - private def zip_producer[A: Type, B: Type](producer1: Producer[A], producer2: Producer[B]) = { + private def zip_producer[A, B](producer1: Producer[A], producer2: Producer[B]) = { new Producer[(A, B)] { type St = (producer1.St, producer2.St) @@ -459,7 +459,7 @@ object Test { } } - def zip[B : Type, C : Type](f: (Expr[A] => Expr[B] => Expr[C]), stream2: Stream[B]): Stream[C] = { + def zip[B: Type, C: Type](f: (Expr[A] => Expr[B] => Expr[C]), stream2: Stream[B]): Stream[C] = { val Stream(stream_b) = stream2 From 070c8a2cd002ebf8a2d6814acbc90ca941e05615 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Fri, 27 Apr 2018 14:32:19 +0200 Subject: [PATCH 22/28] Fix Types --- .../staged-streams_1.scala | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 76ceeeb16ccb..83da9bf31212 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -101,7 +101,7 @@ object Test { * @tparam B the element type of the resulting stream * @return a new stream resulting from applying `f` in the `step` function of the input stream's producer. */ - private def mapRaw[A: Type, B: Type](f: (A => (B => Expr[Unit]) => Expr[Unit]), stream: StagedStream[A]): StagedStream[B] = { + private def mapRaw[A, B](f: (A => (B => Expr[Unit]) => Expr[Unit]), stream: StagedStream[A]): StagedStream[B] = { stream match { case Linear(producer) => { val prod = new Producer[B] { @@ -194,7 +194,7 @@ object Test { * @return the stream with the new producer. If the passed stream was linear, the new termination is added * otherwise the new termination is propagated to all nested ones, recursively. */ - private def addTerminationCondition[A: Type](condition: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { + private def addTerminationCondition[A](condition: Expr[Boolean] => Expr[Boolean], stream: StagedStream[A]): StagedStream[A] = { def addToProducer[A](f: Expr[Boolean] => Expr[Boolean], producer: Producer[A]): Producer[A] = { producer.card match { case Many => @@ -310,8 +310,8 @@ object Test { case (Nested(producer1, nestf1), Linear(producer2)) => mapRaw[(B, Expr[A]), (Expr[A], B)]((t => k => '{ ~k((t._2, t._1)) }), pushLinear[B, _, Expr[A]](producer2, producer1, nestf1)) - case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => ??? - // zipRaw[A, B](Linear(makeLinear(stream1)), stream2) + case (Nested(producer1, nestf1), Nested(producer2, nestf2)) => + zipRaw[A, B](Linear(makeLinear(stream1)), stream2) } } @@ -403,7 +403,7 @@ object Test { } } - private def pushLinear[A: Type, B, C: Type](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { + private def pushLinear[A, B, C](producer: Producer[A], nestedProducer: Producer[B], nestedf: (B => StagedStream[C])): StagedStream[(A, C)] = { val newProducer = new Producer[(Var[Boolean], producer.St, B)] { type St = (Var[Boolean], producer.St, nestedProducer.St) From e2e94841f76d7ef8209f48d0394b384d6234ca21 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Sun, 29 Apr 2018 17:36:17 +0200 Subject: [PATCH 23/28] Implement zip (nested/nested) --- .../staged-streams_1.check | 4 ++- .../staged-streams_1.scala | 33 ++++++++++++++----- 2 files changed, 27 insertions(+), 10 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.check b/tests/run-with-compiler-custom-args/staged-streams_1.check index a9891d75b82a..2dda33f6de1a 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.check +++ b/tests/run-with-compiler-custom-args/staged-streams_1.check @@ -14,4 +14,6 @@ 15 -15 \ No newline at end of file +15 + +36 \ No newline at end of file diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 83da9bf31212..819f6d0a0ec1 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -21,8 +21,6 @@ object Test { } } - type Id[A] = A - trait Producer[A] { self => type St val card: Cardinality @@ -356,10 +354,12 @@ object Test { } else { ~currentAdvance.update('{oldAdvance}) + oldAdvance(_) } - ~currentAdvance.update('{newAdvance}) - newAdvance(_) }} + + ~currentAdvance.update('{newAdvance}) + newAdvance(_) }) } case nested: Nested[A, bt] => @@ -379,7 +379,7 @@ object Test { Var('{ (_: Unit) => ()}){ advf => { Var('{ true }) { hasNext => { Var('{ null.asInstanceOf[A] }) { curr => '{ - val adv: Unit => Unit = { _ => + def adv: Unit => Unit = { _ => ~hasNext.update(producer.hasNext(st)) if(~hasNext.get) { ~producer.step(st, el => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) @@ -395,9 +395,19 @@ object Test { }}) } - def step(st: St, k: Expr[A] => Expr[Unit]): Expr[Unit] = ??? + def step(st: St, k: Expr[A] => Expr[Unit]): Expr[Unit] = { + val (flag, current, advf) = st + var el: Var[A] = current + val f: Expr[Unit => Unit] = advf.get + + f('()) + k((el.get)) + } - def hasNext(st: St): Expr[Boolean] = ??? + def hasNext(st: St): Expr[Boolean] = { + val (flag, _, _) = st + flag.get + } } } } @@ -460,9 +470,7 @@ object Test { } def zip[B: Type, C: Type](f: (Expr[A] => Expr[B] => Expr[C]), stream2: Stream[B]): Stream[C] = { - val Stream(stream_b) = stream2 - Stream(mapRaw[(Expr[A], Expr[B]), Expr[C]]((t => k => '{ ~k(f(t._1)(t._2)) }), zipRaw[A, Expr[B]](stream, stream_b))) } } @@ -548,6 +556,11 @@ object Test { .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)}) ) .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def test10() = Stream + .of('{Array(1, 2, 3)}).flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d + ~dp })) + .zip(((a : Expr[Int]) => (b : Expr[Int]) => '{ ~a + ~b }), Stream.of('{Array(1, 2, 3)}).flatMap((d: Expr[Int]) => Stream.of('{Array(1, 2, 3)}).map((dp: Expr[Int]) => '{ ~d + ~dp })) ) + .fold('{0}, ((a: Expr[Int], b : Expr[Int]) => '{ ~a + ~b })) + def main(args: Array[String]): Unit = { println(test1().run) println @@ -566,6 +579,8 @@ object Test { println(test8().run) println println(test9().run) + println + println(test10().run) } } From ad2f6c84cecb09c48e9ab80c37b969b3a9432e8b Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Tue, 1 May 2018 18:09:38 +0200 Subject: [PATCH 24/28] Add comments --- .../staged-streams_1.scala | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 819f6d0a0ec1..2b33d160efd2 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -21,11 +21,35 @@ object Test { } } + /*** Producer represents a linear production of values. + * + * Conceptually the design of the producer has its roots in `unfold` where a stream is a product type of some state + * and a stepper function. The latter transforms the state and returns either the end-of-the-stream or a value and + * the new state. The existential quantification over the state keeps it private: the only permissible operation is + * to pass it to the step function. + * + * @tparam A type of the collection elements + */ trait Producer[A] { self => type St val card: Cardinality + /** Initialization method that defines new state, if needed by the combinator that this producer defines. + * + * e.g., `addCounter` which adds a counter + * + * @param k the continuation that is invoked after the new state is defined in the body of `init` + * @return expr value of unit per the CPS-encoding + */ def init(k: St => Expr[Unit]): Expr[Unit] + + /** Step method that defines the transformation of data, if applicable. + * + * + * @param st + * @param k + * @return + */ def step(st: St, k: (A => Expr[Unit])): Expr[Unit] def hasNext(st: St): Expr[Boolean] } @@ -183,7 +207,7 @@ object Test { Stream(flatMapRaw[Expr[A], Expr[A]]((a => { Linear(filterStream(a)) }), stream)) } - /** Adds a new termination condition to a producer of cardinality `Many`. + /** Adds a new termination condition to a stream (recursively if nested) of cardinality `Many`. * * @param condition the termination condition as a function accepting the existing condition (the result * of the `hasNext` from the passed `stream`'s producer. From 48ad346dcae53cb30d5bc65c287c90b308033b97 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Wed, 2 May 2018 19:04:06 +0200 Subject: [PATCH 25/28] Bug fix and more comments --- .../staged-streams_1.scala | 44 ++++++++++++++----- 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 2b33d160efd2..9a7932231111 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -21,32 +21,43 @@ object Test { } } - /*** Producer represents a linear production of values. + /*** Producer represents a linear production of values with a loop structure. * * Conceptually the design of the producer has its roots in `unfold` where a stream is a product type of some state - * and a stepper function. The latter transforms the state and returns either the end-of-the-stream or a value and + * and a stepper function: + * + * {{ + * trait Stream[+A] + * case class Unfold[S, +A](state: S, step: (S) => Option[(S, A)]) extends Stream[+A] + * }} + * + * The latter transforms the state and returns either the end-of-the-stream or a value and * the new state. The existential quantification over the state keeps it private: the only permissible operation is - * to pass it to the step function. + * to pass it to the function. However in `Producer` the elements are not pulled but the step accepts a continuation. * - * @tparam A type of the collection elements + * A Producer defines the three basic elements of a loop structure: + * - `init` contributes the code before iteration starts + * - `step` contributes the code during execution + * - `hasNext` contributes the code of the boolean test to end the iteration + * + * @tparam A type of the collection element. Since a `Producer` is polymorphic yet it handles `Expr` values, we + * can pack together fragments of code to accompany each element production (e.g., a variable incremented + * during each transformation) */ trait Producer[A] { self => type St val card: Cardinality /** Initialization method that defines new state, if needed by the combinator that this producer defines. - * - * e.g., `addCounter` which adds a counter * * @param k the continuation that is invoked after the new state is defined in the body of `init` * @return expr value of unit per the CPS-encoding */ def init(k: St => Expr[Unit]): Expr[Unit] - /** Step method that defines the transformation of data, if applicable. + /** Step method that defines the transformation of data. * - * - * @param st + * @param st the state needed for this iteration step * @param k * @return */ @@ -111,10 +122,10 @@ object Test { /** Handles generically the mapping of elements from one producer to another. * `mapRaw` can be potentially used threading quoted values from one stream to another. However - * is can be also used by handling any kind of quoted value. + * is can be also used by declaring any kind of computation we need to perform during each step. * * e.g., `mapRaw[(Var[Int], A), A]` transforms a stream that declares a variable and holds a value in each - * iteration step to a stream that is not aware of the aforementioned variable. + * iteration step, to a stream that is not aware of the aforementioned variable. * * @param f the function to apply at each step. f is of type `(A => (B => Expr[Unit])` where A is the type of * the incoming stream. When applied to an element, `f` returns the continuation for elements of `B` @@ -337,7 +348,12 @@ object Test { } } - /** + /** Make a stream linear + * + * Performs reification of the `stream`. It converts it to a function that will, when called, produce the current element + * and advance the stream -- or report the end-of-stream. + * The reified stream is an imperative *non-recursive* function, called `adv`, of `Unit => Unit` type. Nested streams are + * also handled. * * @param stream * @tparam A @@ -369,6 +385,10 @@ object Test { if(~producer.hasNext(st)) { ~producer.step(st, k) } + else { + val newAdvance = ~currentAdvance.get + newAdvance(_) + } }) case Many => producer.init(st => '{ val oldAdvance : Unit => Unit = ~currentAdvance.get From e949a8a12fa8258982a6dfbd3d55cfb4c7044e22 Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Thu, 3 May 2018 17:10:39 +0200 Subject: [PATCH 26/28] Fix nested zip bug --- .../staged-streams_1.check | 2 +- .../staged-streams_1.scala | 125 +++++++++++++----- 2 files changed, 94 insertions(+), 33 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.check b/tests/run-with-compiler-custom-args/staged-streams_1.check index 2dda33f6de1a..82ff46515903 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.check +++ b/tests/run-with-compiler-custom-args/staged-streams_1.check @@ -16,4 +16,4 @@ 15 -36 \ No newline at end of file +72 \ No newline at end of file diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 9a7932231111..35105f56767a 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -33,14 +33,16 @@ object Test { * * The latter transforms the state and returns either the end-of-the-stream or a value and * the new state. The existential quantification over the state keeps it private: the only permissible operation is - * to pass it to the function. However in `Producer` the elements are not pulled but the step accepts a continuation. + * to pass it to the function. + * + * Note: in `Producer` the elements are not pulled but the step accepts a continuation. * * A Producer defines the three basic elements of a loop structure: * - `init` contributes the code before iteration starts * - `step` contributes the code during execution * - `hasNext` contributes the code of the boolean test to end the iteration * - * @tparam A type of the collection element. Since a `Producer` is polymorphic yet it handles `Expr` values, we + * @tparam A type of the collection element. Since a `Producer` is polymorphic it handles `Expr` values, we * can pack together fragments of code to accompany each element production (e.g., a variable incremented * during each transformation) */ @@ -57,11 +59,17 @@ object Test { /** Step method that defines the transformation of data. * - * @param st the state needed for this iteration step - * @param k - * @return + * @param st the state needed for this iteration step + * @param k the continuation that accepts each element and proceeds with the step-wise processing + * @return expr value of unit per the CPS-encoding */ def step(st: St, k: (A => Expr[Unit])): Expr[Unit] + + /** The condition that checks for termination + * + * @param st the state needed for this iteration check + * @return the expression for a boolean + */ def hasNext(st: St): Expr[Boolean] } @@ -75,9 +83,18 @@ object Test { case class Stream[A: Type](stream: StagedStream[Expr[A]]) { + /** Main consumer + * + * Fold accumulates the results in a variable and delegates its functionality to `foldRaw` + * + * @param z the accumulator + * @param f the zipping function + * @tparam W the type of the accumulator + * @return + */ def fold[W: Type](z: Expr[W], f: ((Expr[W], Expr[A]) => Expr[W])): Expr[W] = { Var(z) { s: Var[W] => '{ - ~fold_raw[Expr[A]]((a: Expr[A]) => '{ + ~foldRaw[Expr[A]]((a: Expr[A]) => '{ ~s.update(f(s.get, a)) }, stream) @@ -86,7 +103,7 @@ object Test { } } - private def fold_raw[A](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { + private def foldRaw[A](consumer: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { stream match { case Linear(producer) => { producer.card match { @@ -105,7 +122,7 @@ object Test { } } case nested: Nested[A, bt] => { - fold_raw[bt](((e: bt) => fold_raw[A](consumer, nested.nestedf(e))), Linear(nested.producer)) + foldRaw[bt](((e: bt) => foldRaw[A](consumer, nested.nestedf(e))), Linear(nested.producer)) } } } @@ -355,6 +372,43 @@ object Test { * The reified stream is an imperative *non-recursive* function, called `adv`, of `Unit => Unit` type. Nested streams are * also handled. * + * @example {{{ + * + * Stream.of(1,2,3).flatMap(d => ...) + * .zip(Stream.of(1,2,3).flatMap(d => ...)) + * .map{ case (a, b) => a + b } + * .fold(0)((a, b) => a + b) + * }}} + * + * --> + * + * {{{ + * /* initialization for stream 1 */ + * + * var curr = null.asInstanceOf[Int] // keeps each element from reified stream + * var nadv: Unit => Unit = (_) => () // keeps the advance for each nested level + * + * def adv: Unit => Unit = /* Linearization of stream1 - updates curr from stream1 */ + * nadv = adv + * adv() + * + * /* initialization for stream 2 */ + * + * def outer () = { + * /* initialization for outer stream of stream 2 */ + * def inner() = { + * /* initialization for inner stream of stream 2 */ + * val el = curr + * nadv() + * /* process elements for map and fold */ + * inner() + * } + * inner() + * outer() + * } + * outer() + * }}} + * * @param stream * @tparam A * @return @@ -366,18 +420,18 @@ object Test { /** Helper function that orchestrates the handling of the function that represents an `advance: Unit => Unit`. * It reifies a nested stream as calls to `advance`. Advance encodes the step function of each nested stream. * It is used in the init of a producer of a nested stream. When an inner stream finishes, the - * `currentAdvance` holds the function to the `advance` function of the earlier stream. + * `nadv` holds the function to the `advance` function of the earlier stream. * `makeAdvanceFunction`, for each nested stream, installs a new `advance` function that after * the stream finishes it will restore the earlier one. * * When `advance` is called the result is consumed in the continuation. Within this continuation * the resulting value should be saved in a variable. * - * @param currentAdvance variable that holds a function that represents the stream at each level. + * @param nadv variable that holds a function that represents the stream at each level. * @param k the continuation that consumes a variable. * @return the quote of the orchestrated code that will be executed as */ - def makeAdvanceFunction[A](currentAdvance: Var[Unit => Unit], k: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { + def makeAdvanceFunction[A](nadv: Var[Unit => Unit], k: A => Expr[Unit], stream: StagedStream[A]): Expr[Unit] = { stream match { case Linear(producer) => producer.card match { @@ -386,28 +440,28 @@ object Test { ~producer.step(st, k) } else { - val newAdvance = ~currentAdvance.get - newAdvance(_) + val f = ~nadv.get + f(()) } }) case Many => producer.init(st => '{ - val oldAdvance : Unit => Unit = ~currentAdvance.get - val newAdvance : Unit => Unit = { _: Unit => { + val oldnadv: Unit => Unit = ~nadv.get + val adv1: Unit => Unit = { _: Unit => { if(~producer.hasNext(st)) { ~producer.step(st, k) } else { - ~currentAdvance.update('{oldAdvance}) - oldAdvance(_) + ~nadv.update('{oldnadv}) + oldnadv(()) } }} - ~currentAdvance.update('{newAdvance}) - newAdvance(_) + ~nadv.update('{adv1}) + adv1(()) }) } case nested: Nested[A, bt] => - makeAdvanceFunction(currentAdvance, (a: bt) => makeAdvanceFunction(currentAdvance, k, nested.nestedf(a)), Linear(nested.producer)) + makeAdvanceFunction(nadv, (a: bt) => makeAdvanceFunction(nadv, k, nested.nestedf(a)), Linear(nested.producer)) } } @@ -420,32 +474,37 @@ object Test { def init(k: St => Expr[Unit]): Expr[Unit] = { producer.init(st => - Var('{ (_: Unit) => ()}){ advf => { + Var('{ (_: Unit) => ()}){ nadv => { Var('{ true }) { hasNext => { Var('{ null.asInstanceOf[A] }) { curr => '{ + + // Code generation of the `adv` function def adv: Unit => Unit = { _ => ~hasNext.update(producer.hasNext(st)) if(~hasNext.get) { - ~producer.step(st, el => makeAdvanceFunction[Expr[A]](advf, (a => curr.update(a)), nestedf(el))) + ~producer.step(st, el => { + makeAdvanceFunction[Expr[A]](nadv, (a => curr.update(a)), nestedf(el)) + }) } } - ~advf.update('{adv}) + ~nadv.update('{adv}) adv(()) - - ~k((hasNext, curr, advf)) + ~k((hasNext, curr, nadv)) }} }} }}) } def step(st: St, k: Expr[A] => Expr[Unit]): Expr[Unit] = { - val (flag, current, advf) = st - var el: Var[A] = current - val f: Expr[Unit => Unit] = advf.get + val (flag, current, nadv) = st + '{ + var el = ~current.get + val f: Unit => Unit = ~nadv.get + f(()) + ~k('(el)) + } - f('()) - k((el.get)) } def hasNext(st: St): Expr[Boolean] = { @@ -465,8 +524,8 @@ object Test { def init(k: St => Expr[Unit]): Expr[Unit] = { producer.init(s1 => '{ ~nestedProducer.init(s2 => - Var('{ ~producer.hasNext(s1) }) { term1r => - k((term1r, s1, s2)) + Var('{ ~producer.hasNext(s1) }) { flag => + k((flag, s1, s2)) })}) } @@ -491,6 +550,7 @@ object Test { }) } + /** Computes the producer of zipping two linear streams **/ private def zip_producer[A, B](producer1: Producer[A], producer2: Producer[B]) = { new Producer[(A, B)] { @@ -513,6 +573,7 @@ object Test { } } + /** zip **/ def zip[B: Type, C: Type](f: (Expr[A] => Expr[B] => Expr[C]), stream2: Stream[B]): Stream[C] = { val Stream(stream_b) = stream2 Stream(mapRaw[(Expr[A], Expr[B]), Expr[C]]((t => k => '{ ~k(f(t._1)(t._2)) }), zipRaw[A, Expr[B]](stream, stream_b))) From 1d8b7893573d1db62f465c693b18afec7cfa850e Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Thu, 3 May 2018 17:15:15 +0200 Subject: [PATCH 27/28] Add note about the port --- tests/run-with-compiler-custom-args/staged-streams_1.scala | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 35105f56767a..07e7283d261c 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -1,6 +1,10 @@ import dotty.tools.dotc.quoted.Toolbox._ import scala.quoted._ +/** + * Port of the strymonas library as described in O. Kiselyov et al., Stream fusion, to completeness (POPL 2017) + */ + object Test { // TODO: remove as it exists in Quoted Lib From 3c46da82067f34e3247f6139a0b59bc01ba5d06f Mon Sep 17 00:00:00 2001 From: Aggelos Biboudis Date: Mon, 28 May 2018 13:18:00 +0200 Subject: [PATCH 28/28] Update comment to reflect curr changing --- .../run-with-compiler-custom-args/staged-streams_1.scala | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/run-with-compiler-custom-args/staged-streams_1.scala b/tests/run-with-compiler-custom-args/staged-streams_1.scala index 07e7283d261c..f5d556615bb0 100644 --- a/tests/run-with-compiler-custom-args/staged-streams_1.scala +++ b/tests/run-with-compiler-custom-args/staged-streams_1.scala @@ -389,12 +389,15 @@ object Test { * {{{ * /* initialization for stream 1 */ * - * var curr = null.asInstanceOf[Int] // keeps each element from reified stream + * var curr = 0.asInstanceOf[Int] // keeps each element from reified stream * var nadv: Unit => Unit = (_) => () // keeps the advance for each nested level * - * def adv: Unit => Unit = /* Linearization of stream1 - updates curr from stream1 */ + * def adv: Unit => Unit = { + * /* Linearization of stream1 - updates curr from stream1 */ + * curr = ... + * } * nadv = adv - * adv() + * nadv() * * /* initialization for stream 2 */ *