Migrate from deprecated scala.util.parsing.json in scala-parser-combinators (see scala/scala-parser-combinators#99) to jackson-module-scala

Author: tashoyan

pom.xml

@@ -53,7 +53,6 @@
         <flink-shaded-jackson.version>2.7.9-5.0</flink-shaded-jackson.version>
         <scopt.version>3.7.1</scopt.version>
         <scalaj-http.version>2.4.1</scalaj-http.version>
-        <scala-parser-combinators.version>1.1.0</scala-parser-combinators.version>
         <scalatest.version>3.0.5</scalatest.version>
         <scalamock.version>4.1.0</scalamock.version>
         <scalactic.version>3.0.5</scalactic.version>
@@ -86,8 +85,10 @@
         <paranamer.version>2.8</paranamer.version>
         <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
         <fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
+        <fasterxml.jackson-module-scala.version>2.6.7.1</fasterxml.jackson-module-scala.version>
         <fasterxml.jackson-databind.version>2.6.7.1</fasterxml.jackson-databind.version>
         <scala-xml.version>1.0.5</scala-xml.version>
+        <scala-parser-combinators.version>1.1.0</scala-parser-combinators.version>
         <commons-compress.version>1.8.1</commons-compress.version>
         <commons-io.version>2.6</commons-io.version>
         <!--json4s version is dictated by Spark-->
@@ -271,9 +272,14 @@
                 <version>${scalaj-http.version}</version>
             </dependency>
             <dependency>
-                <groupId>org.scala-lang.modules</groupId>
-                <artifactId>scala-parser-combinators_${compiler.scala.compat.version}</artifactId>
-                <version>${scala-parser-combinators.version}</version>
+                <groupId>com.fasterxml.jackson.module</groupId>
+                <artifactId>jackson-module-scala_${compiler.scala.compat.version}</artifactId>
+                <version>${fasterxml.jackson-module-scala.version}</version>
+            </dependency>
+            <dependency>
+                <groupId>com.fasterxml.jackson.core</groupId>
+                <artifactId>jackson-databind</artifactId>
+                <version>${fasterxml.jackson-databind.version}</version>
             </dependency>
             <dependency>
                 <groupId>org.slf4j</groupId>
@@ -483,11 +489,6 @@
                 <artifactId>jackson-core</artifactId>
                 <version>${fasterxml.jackson.version}</version>
             </dependency>
-            <dependency>
-                <groupId>com.fasterxml.jackson.core</groupId>
-                <artifactId>jackson-databind</artifactId>
-                <version>${fasterxml.jackson-databind.version}</version>
-            </dependency>
             <dependency>
                 <groupId>com.fasterxml.jackson.core</groupId>
                 <artifactId>jackson-annotations</artifactId>
@@ -503,6 +504,11 @@
                 <artifactId>scala-xml_${compiler.scala.compat.version}</artifactId>
                 <version>${scala-xml.version}</version>
             </dependency>
+            <dependency>
+                <groupId>org.scala-lang.modules</groupId>
+                <artifactId>scala-parser-combinators_${compiler.scala.compat.version}</artifactId>
+                <version>${scala-parser-combinators.version}</version>
+            </dependency>
             <dependency>
                 <groupId>org.lz4</groupId>
                 <artifactId>lz4-java</artifactId>

shared-resources/pom.xml

@@ -23,6 +23,11 @@
             <artifactId>spark-catalyst_${compiler.scala.compat.version}</artifactId>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+            <scope>provided</scope>
+        </dependency>
         <dependency>
             <groupId>org.scala-lang</groupId>
             <artifactId>scala-reflect</artifactId>
@@ -105,8 +110,8 @@
             <scope>test</scope>
         </dependency>
         <dependency>
-            <groupId>org.scala-lang.modules</groupId>
-            <artifactId>scala-parser-combinators_${compiler.scala.compat.version}</artifactId>
+            <groupId>com.fasterxml.jackson.module</groupId>
+            <artifactId>jackson-module-scala_${compiler.scala.compat.version}</artifactId>
             <scope>test</scope>
         </dependency>
     </dependencies>

shared-resources/src/test/scala/com/github/tashoyan/telecom/spark/DataFramesTest.scala

@@ -1,15 +1,16 @@
 package com.github.tashoyan.telecom.spark
 
+import java.io.StringWriter
 import java.sql.Timestamp
 import java.time.{ZoneId, ZonedDateTime}
 
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
 import com.github.tashoyan.telecom.spark.DataFrames.RichDataFrame
 import com.github.tashoyan.telecom.test.SparkTestHarness
 import org.apache.spark.sql.types._
 import org.scalatest.FunSuite
 
-import scala.util.parsing.json.{JSON, JSONObject}
-
 class DataFramesTest extends FunSuite with SparkTestHarness {
 
   test("withJsonColumn") {
@@ -31,9 +32,8 @@ class DataFramesTest extends FunSuite with SparkTestHarness {
     val jsonStr = resultDf.select("json_column")
       .as[String]
       .head()
-    val parsedResult = JSON.parseFull(jsonStr)
-      .get
-      .asInstanceOf[Map[String, Any]]
+
+    val parsedResult = jsonToMap(jsonStr)
     assert(parsedResult("int_column") === 1)
     assert(parsedResult("long_column") === 10L)
     assert(parsedResult("double_column") === 2.5)
@@ -56,15 +56,14 @@ class DataFramesTest extends FunSuite with SparkTestHarness {
       StructField("string_column", StringType),
       StructField("timestamp_column", TimestampType)
     ))
-    val sampleStr = Map[String, Any](
+    val sample = Map[String, Any](
       "int_column" -> 1,
       "long_column" -> 10L,
       "double_column" -> 2.5,
       "string_column" -> "one",
       "timestamp_column" -> "1970-01-01T00:00:01.001Z"
     )
-    val sampleJson = JSONObject(sampleStr)
-      .toString()
+    val sampleJson = mapToJson(sample)
 
     val sampleDf = Seq(
       ("something", sampleJson)
@@ -85,4 +84,19 @@ class DataFramesTest extends FunSuite with SparkTestHarness {
     assert(result.getAs[Timestamp]("timestamp_column") === new Timestamp(1001L))
   }
 
+  private val mapper: ObjectMapper = {
+    new ObjectMapper()
+      .registerModule(DefaultScalaModule)
+  }
+
+  private def jsonToMap(json: String): Map[String, Any] = {
+    mapper.readValue(json, classOf[Map[String, Any]])
+  }
+
+  private def mapToJson(map: Map[String, Any]): String = {
+    val out = new StringWriter()
+    mapper.writeValue(out, map)
+    out.toString
+  }
+
 }