Merge tezos/tezos!14890: Tezt/Cloud: Add the OpenTelemetry stack

Co-authored-by: François Thiré <[email protected]>

Approved-by: Paul Laforgue <[email protected]>
Approved-by: Guillaume B <[email protected]>
Approved-by: François Thiré <[email protected]>

See merge request https://gitlab.com/tezos/tezos/-/merge_requests/14890

Author: Marge Bot

tezt/lib_cloud/cli.ml

@@ -227,3 +227,12 @@ let octez_release =
     ~placeholder:"<tag>"
     ~description:"Use the octez release <tag> instead of local octez binaries."
     ()
+
+let open_telemetry =
+  Clap.flag
+    ~section
+    ~set_long:"open-telemetry"
+    ~unset_long:"no-open-telemetry"
+    ~set_long_synonyms:["otel"]
+    ~description:"Run the Open Telemetry stack"
+    grafana

tezt/lib_cloud/cli.mli

@@ -100,3 +100,6 @@ val os : string
 
 (** The tag of the octez release to be used. *)
 val octez_release : string option
+
+(** Activate the Open Telemetry collector. *)
+val open_telemetry : bool

tezt/lib_cloud/cloud.ml

@@ -88,6 +88,8 @@ type t = {
   website : Web.t option;
   prometheus : Prometheus.t option;
   grafana : Grafana.t option;
+  otel : Otel.t option;
+  jaeger : Jaeger.t option;
   deployement : Deployement.t option;
 }
 
@@ -163,6 +165,8 @@ let shutdown ?exn t =
   let* () =
     Option.fold ~none:Lwt.return_unit ~some:Grafana.shutdown t.grafana
   in
+  let* () = Option.fold ~none:Lwt.return_unit ~some:Otel.shutdown t.otel in
+  let* () = Option.fold ~none:Lwt.return_unit ~some:Jaeger.shutdown t.jaeger in
   let* () =
     Option.fold
       ~none:Lwt.return_unit
@@ -215,9 +219,24 @@ let orchestrator deployement f =
       Lwt.return_some grafana
     else Lwt.return_none
   in
+  let* otel, jaeger =
+    if Env.open_telemetry then
+      let* otel = Otel.run ~jaeger:true in
+      let* jaeger = Jaeger.run () in
+      Lwt.return (Some otel, Some jaeger)
+    else Lwt.return (None, None)
+  in
   Log.info "Post prometheus" ;
   let t =
-    {website; agents; prometheus; grafana; deployement = Some deployement}
+    {
+      website;
+      agents;
+      prometheus;
+      grafana;
+      otel;
+      jaeger;
+      deployement = Some deployement;
+    }
   in
   let sigint = sigint () in
   let main_promise =
@@ -500,6 +519,8 @@ let register ?proxy_files ?vms ~__FILE__ ~title ~tags ?seed f =
           agents = [default_agent];
           website = None;
           grafana = None;
+          otel = None;
+          jaeger = None;
           prometheus = None;
           deployement = None;
         }

tezt/lib_cloud/env.ml

@@ -70,6 +70,8 @@ let dns = Cli.dns
 
 let os = Cli.os
 
+let open_telemetry = Cli.open_telemetry
+
 let docker_image =
   (* In localhost mode, we don't want to interact with GCP. The image is taken
      locally. *)

tezt/lib_cloud/env.mli

@@ -79,6 +79,8 @@ val dns : bool
 
 val os : string
 
+val open_telemetry : bool
+
 val init : unit -> unit Lwt.t
 
 val project_id : unit -> string Lwt.t

tezt/lib_cloud/jaeger.ml

@@ -0,0 +1,40 @@
+(*****************************************************************************)
+(*                                                                           *)
+(* SPDX-License-Identifier: MIT                                              *)
+(* SPDX-FileCopyrightText: 2024 Nomadic Labs <[email protected]>      *)
+(*                                                                           *)
+(*****************************************************************************)
+
+type t = unit
+
+let run () =
+  let* () =
+    Process.run
+      "docker"
+      [
+        "run";
+        "-d";
+        "--network";
+        "host";
+        "--rm";
+        "--name";
+        "jaeger";
+        "-p";
+        "16686:16686";
+        "-p";
+        "14250:14250";
+        "jaegertracing/all-in-one:latest";
+      ]
+  in
+  let is_ready output = String.trim output = "200" in
+  let run () =
+    Process.spawn
+      "curl"
+      ["-s"; "-o"; "/dev/null"; "-w"; "%{http_code}"; "http://localhost:16686/"]
+  in
+  let* _ = Env.wait_process ~is_ready ~run () in
+  Lwt.return ()
+
+let shutdown () =
+  let* () = Docker.kill "jaeger" |> Process.check in
+  Lwt.return_unit

tezt/lib_cloud/jaeger.mli

@@ -0,0 +1,12 @@
+(*****************************************************************************)
+(*                                                                           *)
+(* SPDX-License-Identifier: MIT                                              *)
+(* SPDX-FileCopyrightText: 2024 Nomadic Labs <[email protected]>      *)
+(*                                                                           *)
+(*****************************************************************************)
+
+type t
+
+val run : unit -> t Lwt.t
+
+val shutdown : t -> unit Lwt.t

tezt/lib_cloud/otel.ml

@@ -0,0 +1,105 @@
+(*****************************************************************************)
+(*                                                                           *)
+(* SPDX-License-Identifier: MIT                                              *)
+(* SPDX-FileCopyrightText: 2024 Nomadic Labs <[email protected]>      *)
+(*                                                                           *)
+(*****************************************************************************)
+
+type t = unit
+
+let configuration ~jaeger =
+  let jaeger =
+    if jaeger then
+      {|
+  otlp/jaeger:
+      endpoint: http://localhost:4317
+      tls:
+        insecure: true
+|}
+    else ""
+  in
+  Format.asprintf
+    {|
+receivers:
+  otlp:
+    protocols:
+      http:  
+        endpoint: "0.0.0.0:55681"
+
+exporters:
+
+%s
+
+processors:
+  batch:  # Batch processor to optimize telemetry processing
+    timeout: 5s
+
+service:
+  pipelines:
+    traces:  # Pipeline to process trace data
+      receivers: [otlp]
+      processors: [batch]
+      exporters: [otlp/jaeger]
+
+  telemetry:
+    metrics:
+      address: "0.0.0.0:8888"  # Optional: Expose metrics for the collector itself (Prometheus scrapeable)
+
+  extensions:
+    - health_check
+
+extensions:
+  health_check:
+    endpoint: "localhost:13133"
+|}
+    jaeger
+
+let run ~jaeger =
+  let configuration_file =
+    Filename.get_temp_dir_name () // "otel-config.yaml"
+  in
+  let contents = configuration ~jaeger in
+  write_file configuration_file ~contents ;
+  let* () =
+    Process.run
+      "docker"
+      [
+        "run";
+        "-d";
+        "--rm";
+        "--network";
+        "host";
+        "--name";
+        "otel-collector";
+        "-p";
+        "4317-4318:4317-4318";
+        "-p";
+        "13133:13133";
+        "-p";
+        "55680-55681:55680-55681";
+        "-v";
+        Format.asprintf "%s:/etc/otel/config.yaml" configuration_file;
+        "otel/opentelemetry-collector:latest";
+        "--config";
+        "/etc/otel/config.yaml";
+      ]
+  in
+  let is_ready output = String.trim output = "200" in
+  let run () =
+    Process.spawn
+      "curl"
+      [
+        "-s";
+        "-o";
+        "/dev/null";
+        "-w";
+        "%{http_code}";
+        "http://localhost:13133/healthz";
+      ]
+  in
+  let* _ = Env.wait_process ~is_ready ~run () in
+  Lwt.return ()
+
+let shutdown () =
+  let* () = Docker.kill "otel-collector" |> Process.check in
+  Lwt.return_unit

tezt/lib_cloud/otel.mli

@@ -0,0 +1,12 @@
+(*****************************************************************************)
+(*                                                                           *)
+(* SPDX-License-Identifier: MIT                                              *)
+(* SPDX-FileCopyrightText: 2024 Nomadic Labs <[email protected]>      *)
+(*                                                                           *)
+(*****************************************************************************)
+
+type t
+
+val run : jaeger:bool -> t Lwt.t
+
+val shutdown : t -> unit Lwt.t