Merge branch '0.8.x'

# Conflicts:
#	project/Dependencies.scala
#	project/build.properties
#	project/plugins.sbt
#	rest/src/main/scala/io/udash/rest/openapi/OpenApiMetadata.scala
#	rest/src/test/scala/io/udash/rest/RestTestApi.scala

Author: ghik

guide/backend/src/main/scala/io/udash/web/guide/markdown/MarkdownPagesEndpoint.scala

@@ -1,6 +1,7 @@
 package io.udash.web.guide.markdown
 
 import java.io.{BufferedReader, File, FileReader}
+import java.nio.charset.StandardCharsets
 import java.time.Instant
 import java.util.concurrent.ConcurrentHashMap
 
@@ -19,7 +20,7 @@ final class MarkdownPagesEndpoint(guideResourceBase: String)(implicit ec: Execut
   private val renderedPages = new ConcurrentHashMap[MarkdownPage, (Future[String], Instant)]
 
   private def render(file: File): Future[String] = Future {
-    val reader = new BufferedReader(new FileReader(file))
+    val reader = new BufferedReader(new FileReader(file, StandardCharsets.UTF_8))
     val document = parser.parseReader(reader)
     renderer.render(document)
   }

guide/guide/.js/src/main/assets/pages/rest.md

@@ -441,13 +441,13 @@ explicitly. However, the only reason to use it explicitly is in order to customi
 Body parameters are serialized into `JsonValue` objects.
 See [serialization](#body-parameter-serialization) for more details.
 
-#### `@FormBody`
+##### `@FormBody`
 
 Non-`GET` methods may be annotated with `@FormBody`. This changes serialization of body parameters
 from JSON object to HTTP form, encoded as `application/x-www-form-urlencoded`. Each body parameter
 is then serialized into `PlainValue` rather than `JsonValue`.
 
-#### `@CustomBody`
+##### `@CustomBody`
 
 Methods annotated with `@CustomBody` are required to take **exactly one** body parameter. This body parameter will
 be then serialized directly into `HttpBody`. This makes it possible to fully customize the way HTTP body is built.
@@ -460,6 +460,32 @@ object User extends RestDataCompanion[User]
 @PUT @CustomBody def updateUser(user: User): Future[Unit]
 ```
 
+#### Optional parameters
+
+Instead of `@Query`, `@Header`, `@Cookie` and `@Body`, you can also use `@OptQuery`, `@OptHeader`, `@OptCookie` 
+and `@OptBodyField` to make your parameters explicitly optional. The type of such a parameter must be wrapped into
+an `Option`, `Opt`, `OptArg` or similar option-like wrapper, i.e.
+
+```scala
+@GET def pageContent(@OptQuery lang: Opt[String]): Future[String]
+```
+
+When `Opt.Empty` is passed, this query parameter will simply be omitted in the request.
+
+Note how this is different from simply declaring a (non-optional) parameter typed as `Opt[String]`:
+
+```scala
+@GET def pageContent(@Query lang: Opt[String]): Future[String]
+```
+
+In the example above the framework will simply try to encode `Opt[String]` into
+`PlainValue` (see [serialization](#path-query-header-and-cookie-serialization) for more details).
+This will fail because an `Opt[String]` can't be unambiguously represented as a plain string - compilation
+will fail because of lack of appropriate implicit (`AsRaw/AsReal[PlainValue, Opt[String]]`).
+
+However, when the parameter is explicitly optional, the framework knows that the `Opt` is used to express
+possible lack of this parameter while the actual type of that parameter (that needs to be serialized) is `String`.
+
 ### Prefix methods
 
 Prefix methods are methods that return other REST API traits. They are useful for:

rest/.jvm/src/test/resources/RestTestApi.json

@@ -335,6 +335,15 @@
               "default": "q2def"
             }
           },
+          {
+            "name": "q3",
+            "in": "query",
+            "explode": false,
+            "schema": {
+              "type": "integer",
+              "format": "int32"
+            }
+          },
           {
             "name": "c1",
             "in": "cookie",

rest/src/main/scala/io/udash/rest/annotations.scala

@@ -4,6 +4,7 @@ package rest
 import com.avsystem.commons.annotation.{AnnotationAggregate, defaultsToName}
 import com.avsystem.commons.meta.RealSymAnnotation
 import com.avsystem.commons.rpc._
+import com.avsystem.commons.serialization.optionalParam
 import io.udash.rest.raw._
 
 import scala.annotation.StaticAnnotation
@@ -247,6 +248,62 @@ class Cookie(@defaultsToName override val name: String = RestParamTag.paramName)
 class Body(@defaultsToName override val name: String = RestParamTag.paramName)
   extends rpcName(name) with RestParamTag
 
+/**
+  * Like [[Query]] but indicates that the parameter is optional.
+  * This means that its type must be wrapped into an `Option`, `Opt`, `OptArg`, etc. and empty value
+  * corresponds to an absence of the parameter in the request.
+  *
+  * Also, the macro engine looks for serialization and other implicits directly for the type wrapped in an
+  * `Opt`, `Option`, e.g. `AsRaw/AsReal[PlainValue, Something]` is needed when an optional query parameter has
+  * type `Opt[Something]`.
+  */
+class OptQuery(@defaultsToName name: String = RestParamTag.paramName) extends AnnotationAggregate {
+  @Query(name) @optionalParam
+  final def aggregated: List[StaticAnnotation] = reifyAggregated
+}
+
+/**
+  * Like [[Header]] but indicates that the parameter is optional.
+  * This means that its type must be wrapped into an `Option`, `Opt`, `OptArg`, etc. and empty value
+  * corresponds to an absence of the header in the request.
+  *
+  * Also, the macro engine looks for serialization and other implicits directly for the type wrapped in an
+  * `Opt`, `Option`, e.g. `AsRaw/AsReal[PlainValue, Something]` is needed when an optional header parameter has
+  * type `Opt[Something]`.
+  */
+class OptHeader(name: String) extends AnnotationAggregate {
+  @Header(name) @optionalParam
+  final def aggregated: List[StaticAnnotation] = reifyAggregated
+}
+
+/**
+  * Like [[Cookie]] but indicates that the parameter is optional.
+  * This means that its type must be wrapped into an `Option`, `Opt`, `OptArg`, etc. and empty value
+  * corresponds to an absence of the parameter in the request.
+  *
+  * Also, the macro engine looks for serialization and other implicits directly for the type wrapped in an
+  * `Opt`, `Option`, e.g. `AsRaw/AsReal[PlainValue, Something]` is needed when an optional cookie parameter has
+  * type `Opt[Something]`.
+  */
+class OptCookie(@defaultsToName name: String = RestParamTag.paramName) extends AnnotationAggregate {
+  @Cookie(name) @optionalParam
+  final def aggregated: List[StaticAnnotation] = reifyAggregated
+}
+
+/**
+  * Like [[Body]] (for body field parameters) but indicates that the parameter is optional.
+  * This means that its type must be wrapped into an `Option`, `Opt`, `OptArg`, etc. and empty value
+  * corresponds to an absence of the field in the request body.
+  *
+  * Also, the macro engine looks for serialization and other implicits directly for the type wrapped in an
+  * `Opt`, `Option`, e.g. `AsRaw/AsReal[JsonValue, Something]` is needed when an optional field has
+  * type `Opt[Something]`.
+  */
+class OptBodyField(@defaultsToName name: String = RestParamTag.paramName) extends AnnotationAggregate {
+  @Body(name) @optionalParam
+  final def aggregated: List[StaticAnnotation] = reifyAggregated
+}
+
 /**
   * Base trait for annotations which may be applied on REST API methods (including prefix methods)
   * in order to customize outgoing request on the client side.

rest/src/main/scala/io/udash/rest/openapi/OpenApiMetadata.scala

@@ -2,8 +2,10 @@ package io.udash
 package rest.openapi
 
 import com.avsystem.commons._
+import com.avsystem.commons.annotation.bincompat
 import com.avsystem.commons.meta._
 import com.avsystem.commons.rpc._
+import com.avsystem.commons.serialization.optionalParam
 import io.udash.rest.openapi.adjusters._
 import io.udash.rest.raw._
 import io.udash.rest.{Header => HeaderAnnot, _}
@@ -44,7 +46,6 @@ final case class OpenApiMetadata[T](
   @tagged[BodyMethodTag](whenUntagged = new POST)
   @tagged[SomeBodyTag](whenUntagged = new JsonBody)
   @paramTag[RestParamTag](defaultTag = new Body)
-  @paramTag[RestParamTag](defaultTag = new Body)
   @unmatched(RawRest.NotValidHttpMethod)
   bodyMethods: List[OpenApiBodyOperation[_]]
 ) {
@@ -115,7 +116,7 @@ final case class PathOperation(
 sealed trait OpenApiMethod[T] extends TypedMetadata[T] {
   @reifyName(useRawName = true) def name: String
   @reifyAnnot def methodTag: RestMethodTag
-  @multi @rpcParamMetadata @tagged[NonBodyTag] def parameters: List[OpenApiParameter[_]]
+  @multi @rpcParamMetadata @tagged[NonBodyTag] @allowOptional def parameters: List[OpenApiParameter[_]]
   @multi @reifyAnnot def operationAdjusters: List[OperationAdjuster]
   @multi @reifyAnnot def pathAdjusters: List[PathItemAdjuster]
 
@@ -203,15 +204,15 @@ final case class OpenApiBodyOperation[T](
   operationAdjusters: List[OperationAdjuster],
   pathAdjusters: List[PathItemAdjuster],
   parameters: List[OpenApiParameter[_]],
-  @multi @rpcParamMetadata @tagged[Body] bodyFields: List[OpenApiBodyField[_]],
+  @multi @rpcParamMetadata @tagged[Body] @allowOptional bodyFields: List[OpenApiBodyField[_]],
   @reifyAnnot bodyTypeTag: BodyTypeTag,
   resultType: RestResultType[T]
 ) extends OpenApiOperation[T] {
 
   def requestBody(resolver: SchemaResolver): Opt[RefOr[RequestBody]] =
     if (bodyFields.isEmpty) Opt.Empty else {
       val fields = bodyFields.iterator.map(p => (p.info.name, p.schema(resolver))).toList
-      val requiredFields = bodyFields.collect { case p if !p.info.hasFallbackValue => p.info.name }
+      val requiredFields = bodyFields.collect { case p if !p.info.isOptional => p.info.name }
       val schema = Schema(`type` = DataType.Object, properties = IListMap(fields: _*), required = requiredFields)
       val mediaType = bodyTypeTag match {
         case _: JsonBody => HttpBody.JsonType
@@ -226,14 +227,24 @@ final case class OpenApiBodyOperation[T](
 final case class OpenApiParamInfo[T](
   @reifyName(useRawName = true) name: String,
   @optional @composite whenAbsentInfo: Opt[WhenAbsentInfo[T]],
+  @isAnnotated[optionalParam] optional: Boolean,
   @reifyFlags flags: ParamFlags,
   @infer restSchema: RestSchema[T]
 ) extends TypedMetadata[T] {
-  val whenAbsentValue: Opt[JsonValue] = whenAbsentInfo.flatMap(_.fallbackValue)
-  val hasFallbackValue: Boolean = whenAbsentInfo.fold(flags.hasDefaultValue)(_.fallbackValue.isDefined)
+  val whenAbsentValue: Opt[JsonValue] =
+    if (optional) Opt.Empty
+    else whenAbsentInfo.flatMap(_.fallbackValue)
+
+  val isOptional: Boolean = optional || flags.hasDefaultValue || whenAbsentValue.isDefined
+
+  @bincompat private[rest] def hasFallbackValue: Boolean = isOptional
 
   def schema(resolver: SchemaResolver, withDefaultValue: Boolean): RefOr[Schema] =
     resolver.resolve(restSchema) |> (s => if (withDefaultValue) s.withDefaultValue(whenAbsentValue) else s)
+
+  @bincompat private[rest] def this(
+    name: String, whenAbsentInfo: Opt[WhenAbsentInfo[T]], flags: ParamFlags, restSchema: RestSchema[T]
+  ) = this(name, whenAbsentInfo, false, flags, restSchema)
 }
 
 final case class OpenApiParameter[T](
@@ -253,7 +264,7 @@ final case class OpenApiParameter[T](
     val urlEncodeName = in == Location.Query || in == Location.Cookie
     val name = if(urlEncodeName) URLEncoder.encode(info.name, spaceAsPlus = true) else info.name
     val param = Parameter(name, in,
-      required = pathParam || !info.hasFallbackValue,
+      required = pathParam || !info.isOptional,
       schema = info.schema(resolver, withDefaultValue = !pathParam),
       // repeated query/cookie/header params are not supported for now so ensure `explode` is never assumed to be true
       explode = if (in.defaultStyle.explodeByDefault) OptArg(false) else OptArg.Empty

rest/src/main/scala/io/udash/rest/openapi/RestStructure.scala

@@ -92,7 +92,7 @@ object RestStructure extends AdtMetadataCompanion[RestStructure] {
    */
   @positioned(positioned.here) final case class Record[T](
     @multi @reifyAnnot schemaAdjusters: List[SchemaAdjuster],
-    @adtParamMetadata @multi fields: List[Field[_]],
+    @adtParamMetadata @allowOptional @multi fields: List[Field[_]],
     @composite info: GenCaseInfo[T]
   ) extends RestStructure[T] with Case[T] {
 
@@ -110,7 +110,7 @@ object RestStructure extends AdtMetadataCompanion[RestStructure] {
           val props = caseFieldName.map(cfn => (cfn, RefOr(Schema.enumOf(List(info.rawName))))).iterator ++
             fields.iterator.map(f => (f.info.rawName, f.resolveSchema(resolver)))
           val required = caseFieldName.iterator ++
-            fields.iterator.filterNot(_.hasFallbackValue).map(_.info.rawName)
+            fields.iterator.filterNot(_.optional).map(_.info.rawName)
           RefOr(applyAdjusters(Schema(`type` = DataType.Object,
             properties = IListMap(props.toList: _*),
             required = required.toList
@@ -151,8 +151,10 @@ object RestStructure extends AdtMetadataCompanion[RestStructure] {
   ) extends TypedMetadata[T] {
 
     val fallbackValue: Opt[JsonValue] =
-      (whenAbsentInfo.map(_.fallbackValue) orElse defaultValueInfo.map(_.fallbackValue)).flatten
-    val hasFallbackValue: Boolean = fallbackValue.isDefined
+      if (info.optional) Opt.Empty
+      else (whenAbsentInfo.map(_.fallbackValue) orElse defaultValueInfo.map(_.fallbackValue)).flatten
+
+    val optional: Boolean = info.optional || fallbackValue.isDefined
 
     def resolveSchema(resolver: SchemaResolver): RefOr[Schema] = {
       val bareSchema = resolver.resolve(restSchema).withDefaultValue(fallbackValue)

rest/src/main/scala/io/udash/rest/raw/RestMetadata.scala

@@ -280,7 +280,7 @@ final case class HttpMethodMetadata[T](
   @reifyAnnot methodTag: HttpMethodTag,
   @reifyAnnot bodyTypeTag: BodyTypeTag,
   @composite parametersMetadata: RestParametersMetadata,
-  @multi @tagged[Body] @rpcParamMetadata bodyParams: List[ParamMetadata[_]],
+  @multi @tagged[Body] @rpcParamMetadata @allowOptional bodyParams: List[ParamMetadata[_]],
   @isAnnotated[FormBody] formBody: Boolean,
   @multi @reifyAnnot requestAdjusters: List[RequestAdjuster],
   @multi @reifyAnnot responseAdjusters: List[ResponseAdjuster],
@@ -322,9 +322,9 @@ object HttpResponseType {
 
 final case class RestParametersMetadata(
   @multi @tagged[Path] @rpcParamMetadata pathParams: List[PathParamMetadata[_]],
-  @multi @tagged[Header] @rpcParamMetadata headerParams: List[ParamMetadata[_]],
-  @multi @tagged[Query] @rpcParamMetadata queryParams: List[ParamMetadata[_]],
-  @multi @tagged[Cookie] @rpcParamMetadata cookieParams: List[ParamMetadata[_]]
+  @multi @tagged[Header] @rpcParamMetadata @allowOptional headerParams: List[ParamMetadata[_]],
+  @multi @tagged[Query] @rpcParamMetadata @allowOptional queryParams: List[ParamMetadata[_]],
+  @multi @tagged[Cookie] @rpcParamMetadata @allowOptional cookieParams: List[ParamMetadata[_]]
 ) {
   lazy val headerParamsMap: Map[String, ParamMetadata[_]] =
     headerParams.toMapBy(_.name.toLowerCase)

rest/src/main/scala/io/udash/rest/raw/RestRequest.scala

@@ -22,9 +22,9 @@ object HttpMethod extends AbstractValueEnumCompanion[HttpMethod] {
   */
 final case class RestParameters(
   @multi @tagged[Path] path: List[PlainValue] = Nil,
-  @multi @tagged[Header] headers: IMapping[PlainValue] = IMapping.empty,
-  @multi @tagged[Query] query: Mapping[PlainValue] = Mapping.empty,
-  @multi @tagged[Cookie] cookies: Mapping[PlainValue] = Mapping.empty
+  @multi @tagged[Header] @allowOptional headers: IMapping[PlainValue] = IMapping.empty,
+  @multi @tagged[Query] @allowOptional query: Mapping[PlainValue] = Mapping.empty,
+  @multi @tagged[Cookie] @allowOptional cookies: Mapping[PlainValue] = Mapping.empty
 ) {
   def append(method: RestMethodMetadata[_], otherParameters: RestParameters): RestParameters =
     RestParameters(

rest/src/test/scala/io/udash/rest/RestApiTest.scala

@@ -43,7 +43,8 @@ trait RestApiTestScenarios extends RestApiTest {
   }
 
   test("complex GET") {
-    testCall(_.complexGet(0, "a/ +&", 1, "b/ +&", 2, "ć/ +&", 3, "ó /&f"))
+    testCall(_.complexGet(0, "a/ +&", 1, "b/ +&", 2, "ć/ +&", Opt(3), 4, "ó /&f"))
+    testCall(_.complexGet(0, "a/ +&", 1, "b/ +&", 2, "ć/ +&", Opt.Empty, 3, "ó /&f"))
   }
 
   test("multi-param body POST") {

rest/src/test/scala/io/udash/rest/RestTestApi.scala

@@ -77,17 +77,26 @@ trait RestTestApi {
   @pathDescription("path with a followed by b")
   @description("A really complex GET operation")
   @GET("multi/param") def complexGet(
-    @Path("p1") p1: Int, @description("Very serious path parameter") @title("Stri") @Path p2: String,
-    @Header("X-H1") h1: Int, @Header("X-H2") h2: String,
-    q1: Int, @Query("q=2") @whenAbsent("q2def") q2: String = whenAbsent.value,
-    @Cookie c1: Int, @Cookie("có") c2: String
+    @Path("p1") p1: Int,
+    @description("Very serious path parameter") @title("Stri") @Path p2: String,
+    @Header("X-H1") h1: Int,
+    @Header("X-H2") h2: String,
+    q1: Int,
+    @Query("q=2") @whenAbsent("q2def") q2: String = whenAbsent.value,
+    @OptQuery @whenAbsent(Opt(42)) q3: Opt[Int], // @whenAbsent value must be completely ignored in this case
+    @Cookie c1: Int,
+    @Cookie("có") c2: String
   ): Future[RestEntity]
 
   @POST("multi/param") def multiParamPost(
-    @Path("p1") p1: Int, @Path p2: String,
-    @Header("X-H1") h1: Int, @Header("X-H2") h2: String,
-    @Query q1: Int, @Query("q=2") q2: String,
-    b1: Int, @Body("b\"2") @description("weird body field") b2: String
+    @Path("p1") p1: Int,
+    @Path p2: String,
+    @Header("X-H1") h1: Int,
+    @Header("X-H2") h2: String,
+    @Query q1: Int,
+    @Query("q=2") q2: String,
+    b1: Int,
+    @Body("b\"2") @description("weird body field") b2: String
   ): Future[RestEntity]
 
   @CustomBody
@@ -137,8 +146,8 @@ object RestTestApi extends DefaultRestApiCompanion[RestTestApi] {
     def moreFailingGet: Future[Unit] = throw HttpErrorException(503, "nie")
     def neverGet: Future[Unit] = Future.never
     def getEntity(id: RestEntityId): Future[RestEntity] = Future.successful(RestEntity(id, s"${id.value}-name"))
-    def complexGet(p1: Int, p2: String, h1: Int, h2: String, q1: Int, q2: String, c1: Int, c2: String): Future[RestEntity] =
-      Future.successful(RestEntity(RestEntityId(s"$p1-$h1-$q1-$c1"), s"$p2-$h2-$q2-$c2"))
+    def complexGet(p1: Int, p2: String, h1: Int, h2: String, q1: Int, q2: String, q3: Opt[Int], c1: Int, c2: String): Future[RestEntity] =
+      Future.successful(RestEntity(RestEntityId(s"$p1-$h1-$q1-$c1"), s"$p2-$h2-$q2-${q3.getOrElse(".")}-$c2"))
     def multiParamPost(p1: Int, p2: String, h1: Int, h2: String, q1: Int, q2: String, b1: Int, b2: String): Future[RestEntity] =
       Future.successful(RestEntity(RestEntityId(s"$p1-$h1-$q1-$b1"), s"$p2-$h2-$q2-$b2"))
     def singleBodyPut(entity: RestEntity): Future[String] =

rest/src/test/scala/io/udash/rest/openapi/RestSchemaTest.scala

@@ -4,7 +4,7 @@ package rest.openapi
 import com.avsystem.commons._
 import com.avsystem.commons.misc.{AbstractValueEnum, AbstractValueEnumCompanion, EnumCtx}
 import com.avsystem.commons.serialization.json.JsonStringOutput
-import com.avsystem.commons.serialization.{GenCodec, name, transparent}
+import com.avsystem.commons.serialization.{GenCodec, name, optionalParam, transparent}
 import io.udash.rest.openapi.adjusters.description
 import io.udash.rest.{PolyRestDataCompanion, RestDataCompanion}
 import org.scalatest.funsuite.AnyFunSuite
@@ -25,6 +25,7 @@ class RestSchemaTest extends AnyFunSuite {
   case class KejsKlass(
     @name("integer") @customWa(42) int: Int,
     @description("serious dependency") dep: Dependency,
+    @description("optional thing") @optionalParam opty: Opt[String] = Opt("defaultThatMustBeIgnored"),
     @description("serious string") str: Opt[String] = Opt.Empty
   )
   object KejsKlass extends RestDataCompanion[KejsKlass]
@@ -48,6 +49,10 @@ class RestSchemaTest extends AnyFunSuite {
         |        }
         |      ]
         |    },
+        |    "opty": {
+        |      "type": "string",
+        |      "description": "optional thing"
+        |    },
         |    "str": {
         |      "type": "string",
         |      "description": "serious string",