Dotty with explicit nulls (and flow typing)

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -162,6 +162,7 @@ class ScalaSettings extends Settings.SettingGroup {
 
   // Extremely experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting("-Yno-kind-polymorphism", "Enable kind polymorphism (see https://dotty.epfl.ch/docs/reference/kind-polymorphism.html). Potentially unsound.")
+  val YexplicitNulls: Setting[Boolean] = BooleanSetting("-Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
 
   /** Area-specific debug output */
   val YexplainLowlevel: Setting[Boolean] = BooleanSetting("-Yexplain-lowlevel", "When explaining type errors, show types at a lower level.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -421,6 +421,9 @@ object Contexts {
     def useColors: Boolean =
       base.settings.color.value == "always"
 
+    /** Is the explicit nulls option set? */
+    def explicitNulls: Boolean = base.settings.YexplicitNulls.value
+
     protected def init(outer: Context, origin: Context): this.type = {
       util.Stats.record("Context.fresh")
       _outer = outer

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -4,7 +4,7 @@ package core
 
 import scala.annotation.{threadUnsafe => tu}
 import Types._, Contexts._, Symbols._, SymDenotations._, StdNames._, Names._
-import Flags._, Scopes._, Decorators._, NameOps._, Periods._
+import Flags._, Scopes._, Decorators._, NameOps._, Periods._, NullOpsDecorator._
 import unpickleScala2.Scala2Unpickler.ensureConstructor
 import scala.collection.mutable
 import collection.mutable
@@ -295,8 +295,16 @@ class Definitions {
   @tu lazy val AnyRefAlias: TypeSymbol = enterAliasType(tpnme.AnyRef, ObjectType)
   def AnyRefType: TypeRef = AnyRefAlias.typeRef
 
-    @tu lazy val Object_eq: TermSymbol = enterMethod(ObjectClass, nme.eq, methOfAnyRef(BooleanType), Final)
-    @tu lazy val Object_ne: TermSymbol = enterMethod(ObjectClass, nme.ne, methOfAnyRef(BooleanType), Final)
+    @tu lazy val Object_eq: TermSymbol = {
+      // If explicit nulls is enabled, then we want to allow `(x: String).eq(null)`, so we need
+      // to adjust the signature of `eq` accordingly.
+      enterMethod(ObjectClass, nme.eq, methOfAnyRefOrNull(BooleanType), Final)
+    }
+    @tu lazy val Object_ne: TermSymbol = {
+      // If explicit nulls is enabled, then we want to allow `(x: String).ne(null)`, so we need
+      // to adjust the signature of `ne` accordingly.
+      enterMethod(ObjectClass, nme.ne, methOfAnyRefOrNull(BooleanType), Final)
+    }
     @tu lazy val Object_synchronized: TermSymbol = enterPolyMethod(ObjectClass, nme.synchronized_, 1,
         pt => MethodType(List(pt.paramRefs(0)), pt.paramRefs(0)), Final)
     @tu lazy val Object_clone: TermSymbol = enterMethod(ObjectClass, nme.clone_, MethodType(Nil, ObjectType), Protected)
@@ -336,11 +344,29 @@ class Definitions {
     ScalaPackageClass, tpnme.Nothing, AbstractFinal, List(AnyClass.typeRef))
   def NothingType: TypeRef = NothingClass.typeRef
   @tu lazy val RuntimeNothingModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.Nothing")
-  @tu lazy val NullClass: ClassSymbol = enterCompleteClassSymbol(
-    ScalaPackageClass, tpnme.Null, AbstractFinal, List(ObjectClass.typeRef))
+  @tu lazy val NullClass: ClassSymbol = {
+    val parent = if (ctx.explicitNulls) AnyType else ObjectType
+    enterCompleteClassSymbol(ScalaPackageClass, tpnme.Null, AbstractFinal, parent :: Nil)
+  }
   def NullType: TypeRef = NullClass.typeRef
   @tu lazy val RuntimeNullModuleRef: TermRef = ctx.requiredModuleRef("scala.runtime.Null")
 
+  /** An alias for null values that originate in Java code.
+   *  This type gets special treatment in the Typer. Specifically, `JavaNull` can be selected through:
+   *  e.g.
+   *  ```
+   *  // x: String|Null
+   *  x.length // error: `Null` has no `length` field
+   *  // x2: String|JavaNull
+   *  x2.length // allowed by the Typer, but unsound (might throw NPE)
+   *  ```
+   */
+  lazy val JavaNullAlias: TypeSymbol = {
+    assert(ctx.explicitNulls)
+    enterAliasType(tpnme.JavaNull, NullType)
+  }
+  def JavaNullAliasType: TypeRef = JavaNullAlias.typeRef
+
   @tu lazy val ImplicitScrutineeTypeSym =
     newSymbol(ScalaPackageClass, tpnme.IMPLICITkw, EmptyFlags, TypeBounds.empty).entered
   def ImplicitScrutineeTypeRef: TypeRef = ImplicitScrutineeTypeSym.typeRef
@@ -508,12 +534,16 @@ class Definitions {
   @tu lazy val BoxedNumberClass: ClassSymbol          = ctx.requiredClass("java.lang.Number")
   @tu lazy val ClassCastExceptionClass: ClassSymbol   = ctx.requiredClass("java.lang.ClassCastException")
     @tu lazy val ClassCastExceptionClass_stringConstructor: TermSymbol  = ClassCastExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef StringClass)
+      case List(pt) =>
+        val pt1 = if (ctx.explicitNulls) pt.stripNull else pt
+        pt1 isRef StringClass
       case _ => false
     }).symbol.asTerm
   @tu lazy val ArithmeticExceptionClass: ClassSymbol  = ctx.requiredClass("java.lang.ArithmeticException")
     @tu lazy val ArithmeticExceptionClass_stringConstructor: TermSymbol  = ArithmeticExceptionClass.info.member(nme.CONSTRUCTOR).suchThat(_.info.firstParamTypes match {
-      case List(pt) => (pt isRef StringClass)
+      case List(pt) =>
+        val pt1 = if (ctx.explicitNulls) pt.stripNull else pt
+        pt1 isRef StringClass
       case _ => false
     }).symbol.asTerm
 
@@ -783,10 +813,29 @@ class Definitions {
   @tu lazy val InfixAnnot: ClassSymbol = ctx.requiredClass("scala.annotation.infix")
   @tu lazy val AlphaAnnot: ClassSymbol = ctx.requiredClass("scala.annotation.alpha")
 
+  // A list of annotations that are commonly used to indicate that a field/method argument or return
+  // type is not null. These annotations are used by the nullification logic in JavaNullInterop to
+  // improve the precision of type nullification.
+  // We don't require that any of these annotations be present in the class path, but we want to
+  // create Symbols for the ones that are present, so they can be checked during nullification.
+  @tu lazy val NotNullAnnots: List[ClassSymbol] = ctx.getClassesIfDefined(
+    "javax.annotation.Nonnull" ::
+    "edu.umd.cs.findbugs.annotations.NonNull" ::
+    "androidx.annotation.NonNull" ::
+    "android.support.annotation.NonNull" ::
+    "android.annotation.NonNull" ::
+    "com.android.annotations.NonNull" ::
+    "org.eclipse.jdt.annotation.NonNull" ::
+    "org.checkerframework.checker.nullness.qual.NonNull" ::
+    "org.checkerframework.checker.nullness.compatqual.NonNullDecl" ::
+    "org.jetbrains.annotations.NotNull" ::
+    "lombok.NonNull" ::
+    "io.reactivex.annotations.NonNull" :: Nil map PreNamedString)
+
   // convenient one-parameter method types
   def methOfAny(tp: Type): MethodType = MethodType(List(AnyType), tp)
   def methOfAnyVal(tp: Type): MethodType = MethodType(List(AnyValType), tp)
-  def methOfAnyRef(tp: Type): MethodType = MethodType(List(ObjectType), tp)
+  def methOfAnyRefOrNull(tp: Type): MethodType = MethodType(List(ObjectType.maybeNullable), tp)
 
   // Derived types
 
@@ -947,8 +996,16 @@ class Definitions {
       name.drop(prefix.length).forall(_.isDigit))
 
   def isBottomClass(cls: Symbol): Boolean =
-    cls == NothingClass || cls == NullClass
+    if (ctx.explicitNulls && !ctx.phase.erasedTypes) cls == NothingClass
+    else isBottomClassAfterErasure(cls)
+
+  def isBottomClassAfterErasure(cls: Symbol): Boolean = cls == NothingClass || cls == NullClass
+
   def isBottomType(tp: Type): Boolean =
+    if (ctx.explicitNulls && !ctx.phase.erasedTypes) tp.derivesFrom(NothingClass)
+    else isBottomTypeAfterErasure(tp)
+
+  def isBottomTypeAfterErasure(tp: Type): Boolean =
     tp.derivesFrom(NothingClass) || tp.derivesFrom(NullClass)
 
   /** Is a function class.
@@ -1292,18 +1349,22 @@ class Definitions {
   // ----- Initialization ---------------------------------------------------
 
   /** Lists core classes that don't have underlying bytecode, but are synthesized on-the-fly in every reflection universe */
-  @tu lazy val syntheticScalaClasses: List[TypeSymbol] = List(
-    AnyClass,
-    AnyRefAlias,
-    AnyKindClass,
-    andType,
-    orType,
-    RepeatedParamClass,
-    ByNameParamClass2x,
-    AnyValClass,
-    NullClass,
-    NothingClass,
-    SingletonClass)
+  @tu lazy val syntheticScalaClasses: List[TypeSymbol] = {
+    val synth = List(
+      AnyClass,
+      AnyRefAlias,
+      AnyKindClass,
+      andType,
+      orType,
+      RepeatedParamClass,
+      ByNameParamClass2x,
+      AnyValClass,
+      NullClass,
+      NothingClass,
+      SingletonClass)
+
+    if (ctx.explicitNulls) synth :+ JavaNullAlias else synth
+  }
 
   @tu lazy val syntheticCoreClasses: List[Symbol] = syntheticScalaClasses ++ List(
     EmptyPackageVal,

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -450,8 +450,12 @@ object Flags {
    *  is completed)
    */
   val AfterLoadFlags: FlagSet = commonFlags(
-    FromStartFlags, AccessFlags, Final, AccessorOrSealed, LazyOrTrait, SelfName, JavaDefined)
-
+    FromStartFlags, AccessFlags, Final, AccessorOrSealed, LazyOrTrait, SelfName, JavaDefined,
+    // We would like to add JavaEnumValue to this set so that we can correctly
+    // detect it in JavaNullInterop. However, JavaEnumValue is not initialized at this
+    // point, so we just make sure that all the "primitive" flags contained in JavaEnumValue
+    // are mentioned here as well.
+    Enum, StableRealizable)
 
   /** A value that's unstable unless complemented with a Stable flag */
   val UnstableValueFlags: FlagSet = Mutable | Method

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -0,0 +1,156 @@
+package dotty.tools.dotc.core
+
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Flags.JavaDefined
+import dotty.tools.dotc.core.StdNames.{jnme, nme}
+import dotty.tools.dotc.core.Symbols._
+import dotty.tools.dotc.core.Types._
+import NullOpsDecorator._
+
+/** This module defines methods to interpret types of Java symbols, which are implicitly nullable in Java,
+ *  as Scala types, which are explicitly nullable.
+ *
+ *  The transformation is (conceptually) a function `n` that adheres to the following rules:
+ *    (1) n(T)              = T|JavaNull              if T is a reference type
+ *    (2) n(T)              = T                       if T is a value type
+ *    (3) n(C[T])           = C[T]|JavaNull           if C is Java-defined
+ *    (4) n(C[T])           = C[n(T)]|JavaNull        if C is Scala-defined
+ *    (5) n(A|B)            = n(A)|n(B)|JavaNull
+ *    (6) n(A&B)            = n(A) & n(B)
+ *    (7) n((A1, ..., Am)R) = (n(A1), ..., n(Am))n(R) for a method with arguments (A1, ..., Am) and return type R
+ *    (8) n(T)              = T                       otherwise
+ *
+ *   Treatment of generics (rules 3 and 4):
+ *     - if `C` is Java-defined, then `n(C[T]) = C[T]|JavaNull`. That is, we don't recurse
+ *       on the type argument, and only add JavaNull on the outside. This is because
+ *       `C` itself will be nullified, and in particular so will be usages of `C`'s type argument within C's body.
+ *       e.g. calling `get` on a `java.util.List[String]` already returns `String|Null` and not `String`, so
+ *       we don't need to write `java.util.List[String|Null]`.
+ *     - if `C` is Scala-defined, however, then we want `n(C[T]) = C[n(T)]|JavaNull`. This is because
+ *       `C` won't be nullified, so we need to indicate that its type argument is nullable.
+ *
+ *   Notice that since the transformation is only applied to types attached to Java symbols, it doesn't need
+ *   to handle the full spectrum of Scala types. Additionally, some kinds of symbols like constructors and
+ *   enum instances get special treatment.
+ */
+object JavaNullInterop {
+
+  /** Transforms the type `tp` of Java member `sym` to be explicitly nullable.
+   *  `tp` is needed because the type inside `sym` might not be set when this method is called.
+   *
+   *  e.g. given a Java method
+   *  String foo(String arg) { return arg; }
+   *
+   *  After calling `nullifyMember`, Scala will see the method as
+   *
+   *  def foo(arg: String|JavaNull): String|JavaNull
+   *
+   *  This nullability function uses `JavaNull` instead of vanilla `Null`, for usability.
+   *  This means that we can select on the return of `foo`:
+   *
+   *  val len = foo("hello").length
+   *
+   *  But the selection can throw an NPE if the returned value is `null`.
+   */
+  def nullifyMember(sym: Symbol, tp: Type)(implicit ctx: Context): Type = {
+    assert(ctx.explicitNulls)
+    assert(sym.is(JavaDefined), "can only nullify java-defined members")
+
+    // Some special cases when nullifying the type
+    if (sym.name == nme.TYPE_ || sym.isAllOf(Flags.JavaEnumValue))
+      // Don't nullify the `TYPE` field in every class and Java enum instances
+      tp
+    else if (sym.name == nme.toString_ || sym.isConstructor || hasNotNullAnnot(sym))
+      // Don't nullify the return type of the `toString` method.
+      // Don't nullify the return type of constructors.
+      // Don't nullify the return type of methods with a not-null annotation.
+      nullifyExceptReturnType(tp)
+    else
+      // Otherwise, nullify everything
+      nullifyType(tp)
+  }
+
+  private def hasNotNullAnnot(sym: Symbol)(implicit ctx: Context): Boolean =
+    ctx.definitions.NotNullAnnots.exists(nna => sym.unforcedAnnotation(nna).isDefined)
+
+  /** If tp is a MethodType, the parameters and the inside of return type are nullified,
+   *  but the result return type is not nullable.
+   *  If tp is a type of a field, the inside of the type is nullified,
+   *  but the result type is not nullable.
+   */
+  private def nullifyExceptReturnType(tp: Type)(implicit ctx: Context): Type =
+    new JavaNullMap(true)(ctx)(tp)
+
+  /** Nullifies a Java type by adding `| JavaNull` in the relevant places. */
+  private def nullifyType(tp: Type)(implicit ctx: Context): Type =
+    new JavaNullMap(false)(ctx)(tp)
+
+  /** A type map that implements the nullification function on types. Given a Java-sourced type, this adds `| JavaNull`
+   *  in the right places to make the nulls explicit in Scala.
+   *
+   *  @param outermostLevelAlreadyNullable whether this type is already nullable at the outermost level.
+   *                                       For example, `Array[String]|JavaNull` is already nullable at the
+   *                                       outermost level, but `Array[String|JavaNull]` isn't.
+   *                                       If this parameter is set to true, then the types of fields, and the return
+   *                                       types of methods will not be nullified.
+   *                                       This is useful for e.g. constructors, and also so that `A & B` is nullified
+   *                                       to `(A & B) | JavaNull`, instead of `(A|JavaNull & B|JavaNull) | JavaNull`.
+   */
+  private class JavaNullMap(var outermostLevelAlreadyNullable: Boolean)(implicit ctx: Context) extends TypeMap {
+    /** Should we nullify `tp` at the outermost level? */
+    def needsNull(tp: Type): Boolean =
+      !outermostLevelAlreadyNullable && (tp match {
+        case tp: TypeRef =>
+          // We don't modify value types because they're non-nullable even in Java.
+          !tp.symbol.isValueClass &&
+          // We don't modify `Any` because it's already nullable.
+          !tp.isRef(defn.AnyClass) &&
+          // We don't nullify Java varargs at the top level.
+          // Example: if `setNames` is a Java method with signature `void setNames(String... names)`,
+          // then its Scala signature will be `def setNames(names: (String|JavaNull)*): Unit`.
+          // This is because `setNames(null)` passes as argument a single-element array containing the value `null`,
+          // and not a `null` array.
+          !tp.isRef(defn.RepeatedParamClass)
+        case _ => true
+      })
+
+    override def apply(tp: Type): Type = {
+      // Fast version of Type::toJavaNullableUnion that doesn't check whether the type
+      // is already a union.
+      def toJavaNullableUnion(tpe: Type): Type = OrType(tpe, defn.JavaNullAliasType)
+
+      tp match {
+        case tp: TypeRef if needsNull(tp) => toJavaNullableUnion(tp)
+        case appTp @ AppliedType(tycon, targs) =>
+          val oldOutermostNullable = outermostLevelAlreadyNullable
+          // We don't make the outmost levels of type arguements nullable if tycon is Java-defined.
+          // This is because Java classes are _all_ nullified, so both `java.util.List[String]` and
+          // `java.util.List[String|Null]` contain nullable elements.
+          outermostLevelAlreadyNullable = tp.classSymbol.is(JavaDefined)
+          val targs2 = targs map this
+          outermostLevelAlreadyNullable = oldOutermostNullable
+          val appTp2 = derivedAppliedType(appTp, tycon, targs2)
+          if (needsNull(tycon)) toJavaNullableUnion(appTp2) else appTp2
+        case ptp: PolyType =>
+          derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
+        case mtp: MethodType =>
+          val oldOutermostNullable = outermostLevelAlreadyNullable
+          outermostLevelAlreadyNullable = false
+          val paramInfos2 = mtp.paramInfos map this
+          outermostLevelAlreadyNullable = oldOutermostNullable
+          derivedLambdaType(mtp)(paramInfos2, this(mtp.resType))
+        case tp: TypeAlias => mapOver(tp)
+        case tp: AndType =>
+          // nullify(A & B) = (nullify(A) & nullify(B)) | JavaNull, but take care not to add
+          // duplicate `JavaNull`s at the outermost level inside `A` and `B`.
+          outermostLevelAlreadyNullable = true
+          toJavaNullableUnion(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
+        case tp: TypeParamRef if needsNull(tp) => toJavaNullableUnion(tp)
+        // In all other cases, return the type unchanged.
+        // In particular, if the type is a ConstantType, then we don't nullify it because it is the
+        // type of a final non-nullable field.
+        case _ => tp
+      }
+    }
+  }
+}