Use local httpbin to run tests

Fix com-lihaoyi#132

Start a local httpbin container to run tests against.

Https requests are still using remote one since it's hard to setup local
secure ssl. The requests using https are simple and during my tests I
haven't seen any timeout.

Author: wb14123

build.sc

@@ -40,7 +40,8 @@ trait RequestsModule extends CrossScalaModule with PublishModule with Mima {
   object test extends ScalaTests with TestModule.Utest {
     def ivyDeps = Agg(
       ivy"com.lihaoyi::utest::0.7.10",
-      ivy"com.lihaoyi::ujson::1.3.13"
+      ivy"com.lihaoyi::ujson::1.3.13",
+      ivy"com.dimafeng::testcontainers-scala-core:0.41.3"
     )
   }
 }

requests/test/src/requests/RequestTests.scala

@@ -1,9 +1,25 @@
 package requests
 
+import com.dimafeng.testcontainers.GenericContainer
+import org.testcontainers.containers.wait.strategy.Wait
 import utest._
 import ujson._
 
 object RequestTests extends TestSuite{
+
+  val containerDef = GenericContainer.Def(
+    "kennethreitz/httpbin",
+    exposedPorts = Seq(80),
+    waitStrategy = Wait.forHttp("/")
+  )
+  val container = containerDef.start()
+  val localHttpbinHost = container.containerIpAddress
+  val localHttpbin = s"${localHttpbinHost}:${container.mappedPort(80)}"
+
+  override def utestAfterAll(): Unit = {
+    container.stop()
+  }
+
   val tests = Tests{
     test("matchingMethodWorks"){
       val requesters = Seq(
@@ -13,18 +29,18 @@ object RequestTests extends TestSuite{
         requests.put
       )
 
-      for(protocol <- Seq("http", "https")){
+      for(baseUrl <- Seq(s"http://$localHttpbin", "https://httpbin.org")){
         for(r <- requesters){
           for(r2 <- requesters){
-            val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}", check = false)
+            val res = r(s"$baseUrl/${r2.verb.toLowerCase()}", check = false)
             if (r.verb == r2.verb) assert(res.statusCode == 200)
             else assert(res.statusCode == 405)
 
             if (r.verb == r2.verb){
-              val res = r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}")
+              val res = r(s"$baseUrl/${r2.verb.toLowerCase()}")
               assert(res.statusCode == 200)
             }else intercept[RequestFailedException]{
-              r(s"$protocol://httpbin.org/${r2.verb.toLowerCase()}")
+              r(s"$baseUrl/${r2.verb.toLowerCase()}")
             }
           }
         }
@@ -34,26 +50,26 @@ object RequestTests extends TestSuite{
     test("params"){
       test("get"){
         // All in URL
-        val res1 = requests.get("https://httpbin.org/get?hello=world&foo=baz").text()
+        val res1 = requests.get(s"http://$localHttpbin/get?hello=world&foo=baz").text()
         assert(read(res1).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // All in params
         val res2 = requests.get(
-          "https://httpbin.org/get",
+          s"http://$localHttpbin/get",
           params = Map("hello" -> "world", "foo" -> "baz")
         )
         assert(read(res2).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // Mixed URL and params
         val res3 = requests.get(
-          "https://httpbin.org/get?hello=world",
+          s"http://$localHttpbin/get?hello=world",
           params = Map("foo" -> "baz")
         ).text()
         assert(read(res3).obj("args") == Obj("foo" -> "baz", "hello" -> "world"))
 
         // Needs escaping
         val res4 = requests.get(
-          "https://httpbin.org/get?hello=world",
+          s"http://$localHttpbin/get?hello=world",
           params = Map("++-- lol" -> " !@#$%")
         )
         assert(read(res4).obj("args") == Obj("++-- lol" -> " !@#$%", "hello" -> "world"))
@@ -63,7 +79,7 @@ object RequestTests extends TestSuite{
     test("multipart"){
       for(chunkedUpload <- Seq(true, false)) {
         val response = requests.post(
-          "http://httpbin.org/post",
+          s"http://$localHttpbin/post",
           data = MultiPart(
             MultiItem("file1", "Hello!".getBytes, "foo.txt"),
             MultiItem("file2", "Goodbye!")
@@ -79,76 +95,77 @@ object RequestTests extends TestSuite{
     test("cookies"){
       test("session"){
         val s = requests.Session(cookieValues = Map("hello" -> "world"))
-        val res1 = s.get("https://httpbin.org/cookies").text().trim
+        val res1 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj("hello" -> "world")))
-        s.get("https://httpbin.org/cookies/set?freeform=test")
-        val res2 = s.get("https://httpbin.org/cookies").text().trim
+        s.get(s"http://$localHttpbin/cookies/set?freeform=test")
+        val res2 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test", "hello" -> "world")))
       }
       test("raw"){
-        val res1 = requests.get("https://httpbin.org/cookies").text().trim
+        val res1 = requests.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj()))
-        requests.get("https://httpbin.org/cookies/set?freeform=test")
-        val res2 = requests.get("https://httpbin.org/cookies").text().trim
+        requests.get(s"http://$localHttpbin/cookies/set?freeform=test")
+        val res2 = requests.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj()))
       }
       test("space"){
         val s = requests.Session(cookieValues = Map("hello" -> "hello, world"))
-        val res1 = s.get("https://httpbin.org/cookies").text().trim
+        val res1 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res1) == Obj("cookies" -> Obj("hello" -> "hello, world")))
-        s.get("https://httpbin.org/cookies/set?freeform=test+test")
-        val res2 = s.get("https://httpbin.org/cookies").text().trim
+        s.get(s"http://$localHttpbin/cookies/set?freeform=test+test")
+        val res2 = s.get(s"http://$localHttpbin/cookies").text().trim
         assert(read(res2) == Obj("cookies" -> Obj("freeform" -> "test test", "hello" -> "hello, world")))
       }
     }
 
     test("redirects"){
       test("max"){
-        val res1 = requests.get("https://httpbin.org/absolute-redirect/4")
+        val res1 = requests.get(s"http://$localHttpbin/absolute-redirect/4")
         assert(res1.statusCode == 200)
-        val res2 = requests.get("https://httpbin.org/absolute-redirect/5")
+        val res2 = requests.get(s"http://$localHttpbin/absolute-redirect/5")
         assert(res2.statusCode == 200)
-        val res3 = requests.get("https://httpbin.org/absolute-redirect/6", check = false)
+        val res3 = requests.get(s"http://$localHttpbin/absolute-redirect/6", check = false)
         assert(res3.statusCode == 302)
-        val res4 = requests.get("https://httpbin.org/absolute-redirect/6", maxRedirects = 10)
+        val res4 = requests.get(s"http://$localHttpbin/absolute-redirect/6", maxRedirects = 10)
         assert(res4.statusCode == 200)
       }
       test("maxRelative"){
-        val res1 = requests.get("https://httpbin.org/relative-redirect/4")
+        val res1 = requests.get(s"http://$localHttpbin/relative-redirect/4")
         assert(res1.statusCode == 200)
-        val res2 = requests.get("https://httpbin.org/relative-redirect/5")
+        val res2 = requests.get(s"http://$localHttpbin/relative-redirect/5")
         assert(res2.statusCode == 200)
-        val res3 = requests.get("https://httpbin.org/relative-redirect/6", check = false)
+        val res3 = requests.get(s"http://$localHttpbin/relative-redirect/6", check = false)
         assert(res3.statusCode == 302)
-        val res4 = requests.get("https://httpbin.org/relative-redirect/6", maxRedirects = 10)
+        val res4 = requests.get(s"http://$localHttpbin/relative-redirect/6", maxRedirects = 10)
         assert(res4.statusCode == 200)
       }
     }
 
     test("test_reproduction"){
-      requests.get("http://httpbin.org/status/304").text()
+      requests.get(s"http://$localHttpbin/status/304").text()
 
     }
     test("streaming"){
-      val res1 = requests.get("http://httpbin.org/stream/5").text()
+      val res1 = requests.get(s"http://$localHttpbin/stream/5").text()
       assert(res1.linesIterator.length == 5)
-      val res2 = requests.get("http://httpbin.org/stream/52").text()
+      val res2 = requests.get(s"http://$localHttpbin/stream/52").text()
       assert(res2.linesIterator.length == 52)
     }
 
     test("timeouts"){
       test("read"){
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/1", readTimeout = 10)
+          requests.get(s"http://$localHttpbin/delay/1", readTimeout = 10)
         }
-        requests.get("https://httpbin.org/delay/1", readTimeout = 2000)
+        requests.get(s"http://$localHttpbin/delay/1", readTimeout = 2000)
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/3", readTimeout = 2000)
+          requests.get(s"http://$localHttpbin/delay/3", readTimeout = 2000)
         }
       }
       test("connect"){
         intercept[TimeoutException] {
-          requests.get("https://httpbin.org/delay/1", connectTimeout = 1)
+          // use remote httpbin.org so it needs more time to connect
+          requests.get(s"https://httpbin.org/delay/1", connectTimeout = 1)
         }
       }
     }
@@ -167,38 +184,38 @@ object RequestTests extends TestSuite{
     }
 
     test("decompress"){
-      val res1 = requests.get("https://httpbin.org/gzip")
-      assert(read(res1.text()).obj("headers").obj("Host").str == "httpbin.org")
+      val res1 = requests.get(s"http://$localHttpbin/gzip")
+      assert(read(res1.text()).obj("headers").obj("Host").str == localHttpbin)
 
-      val res2 = requests.get("https://httpbin.org/deflate")
-      assert(read(res2).obj("headers").obj("Host").str == "httpbin.org")
+      val res2 = requests.get(s"http://$localHttpbin/deflate")
+      assert(read(res2).obj("headers").obj("Host").str == localHttpbin)
 
-      val res3 = requests.get("https://httpbin.org/gzip", autoDecompress = false)
+      val res3 = requests.get(s"http://$localHttpbin/gzip", autoDecompress = false)
       assert(res3.bytes.length < res1.bytes.length)
 
-      val res4 = requests.get("https://httpbin.org/deflate", autoDecompress = false)
+      val res4 = requests.get(s"http://$localHttpbin/deflate", autoDecompress = false)
       assert(res4.bytes.length < res2.bytes.length)
 
       (res1.bytes.length, res2.bytes.length, res3.bytes.length, res4.bytes.length)
     }
 
     test("compression"){
       val res1 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.None,
         data = new RequestBlob.ByteSourceRequestBlob("Hello World")
       )
       assert(res1.text().contains(""""Hello World""""))
 
       val res2 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.Gzip,
         data = new RequestBlob.ByteSourceRequestBlob("I am cow")
       )
       assert(read(new String(res2.bytes))("data").toString.contains("data:application/octet-stream;base64,H4sIAAAAAA"))
 
       val res3 = requests.post(
-        "https://httpbin.org/post",
+        s"http://$localHttpbin/post",
         compress = requests.Compress.Deflate,
         data = new RequestBlob.ByteSourceRequestBlob("Hear me moo")
       )
@@ -208,7 +225,7 @@ object RequestTests extends TestSuite{
 
     test("headers"){
       test("default"){
-        val res = requests.get("https://httpbin.org/headers").text()
+        val res = requests.get(s"http://$localHttpbin/headers").text()
         val hs = read(res)("headers").obj
         assert(hs("User-Agent").str == "requests-scala")
         assert(hs("Accept-Encoding").str == "gzip, deflate")
@@ -306,7 +323,7 @@ object RequestTests extends TestSuite{
     // to the server. This preserves the 0.8.x behavior, and can always be overriden
     // by passing a comma-separated list of headers instead
     test("duplicateHeaders"){
-      val res = requests.get("https://httpbin.org/get", headers = Seq("x-y" -> "a", "x-y" -> "b"))
+      val res = requests.get(s"http://$localHttpbin/get", headers = Seq("x-y" -> "a", "x-y" -> "b"))
       assert(ujson.read(res)("headers")("X-Y") == Str("b")) // make sure it's not "a,b"
     }
   }