Use EI TensorRT library v2.0.0

Makefile

@@ -76,16 +76,22 @@ endif
 
 ifeq (${TARGET_JETSON_ORIN},1)
 TARGET_JETSON_COMMON=1
-TENSORRT_VERSION=8.5.2
+TENSORRT_VERSION?=8.5.2
+USE_FULL_TFLITE=1
+TARGET_LINUX_AARCH64=1
 endif
 
 ifeq (${TARGET_JETSON_NANO},1)
 TARGET_JETSON=1
+USE_FULL_TFLITE=1
+TARGET_LINUX_AARCH64=1
 endif
 
 ifeq (${TARGET_JETSON},1)
 TARGET_JETSON_COMMON=1
 TENSORRT_VERSION?=8
+USE_FULL_TFLITE=1
+TARGET_LINUX_AARCH64=1
 endif
 
 ifeq (${USE_ETHOS},1)
@@ -165,14 +171,14 @@ endif # USE_MEMRYX
 ifeq (${TARGET_JETSON_COMMON},1)
 TENSORRT_VERSION ?=8
 $(info TENSORRT_VERSION is ${TENSORRT_VERSION})
-ifeq (${TENSORRT_VERSION},8.5.2)
+ifeq (${TENSORRT_VERSION},8.6.2)
+TRT_LDFLAGS += -lei_debug -Ltflite/linux-jetson-nano/trt8.6.2/
+else ifeq (${TENSORRT_VERSION},8.5.2)
 TRT_LDFLAGS += -lei_debug -Ltflite/linux-jetson-nano/trt8.5.2/
 else ifeq (${TENSORRT_VERSION},8)
 TRT_LDFLAGS += -lei_debug -Ltflite/linux-jetson-nano/trt8/
-else ifeq (${TENSORRT_VERSION},7)
-TRT_LDFLAGS += -lei_debug7 -Ltflite/linux-jetson-nano/trt7/
 else
-$(error Invalid TensorRT version - supported versions are 7 and 8.)
+$(error Invalid TensorRT version)
 endif # TENSORRT_VERSION
 TRT_LDFLAGS += -lcudart -lnvinfer -lnvonnxparser
 LDFLAGS += $(TRT_LDFLAGS) -lstdc++fs -Ltflite/linux-jetson-nano/ -Wl,--warn-unresolved-symbols,--unresolved-symbols=ignore-in-shared-libs

tflite/linux-jetson-nano/README.md

@@ -11,6 +11,3 @@ $ sh docker/build.sh --file docker/ubuntu-cross-aarch64.Dockerfile --tag tensorr
 ```
 
 You can find the library in `/workspace/TensorRT/build/out/libei_debug.a` in the container. It is also automatically copied after a successful build to the /docker folder.
-
-`libei_debug7.a` is a version of the library built with TensorRT 7.x.
-

tflite/linux-jetson-nano/download.sh

@@ -3,21 +3,22 @@ set -e
 
 SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )"
 
-TRT7="tensorrt-shared-libs.zip"
-TRT8="shared-libs-jetpack4.6.4.zip"
-TRT8_5_2="shared-libs-jetpack5.1.2.zip"
+TRT8="shared-libs-jetpack4.6.x.zip"
+TRT8_5_2="shared-libs-jetpack5.1.x.zip"
+TRT8_6_2="shared-libs-jetpack6.0.0.zip"
 
 download_lib() {
     dir="$1"
     zip_file="$2"
 
+    mkdir -p "$dir"
     cd "$dir"
     rm -rf *.so
     wget https://cdn.edgeimpulse.com/build-system/"$zip_file"
     unzip -q "$zip_file"
     rm -rf "$zip_file"
 }
 
-download_lib "$SCRIPTPATH"/trt7 "$TRT7"
 download_lib "$SCRIPTPATH"/trt8 "$TRT8"
-download_lib "$SCRIPTPATH"/trt8.5.2 "$TRT8_5_2"
+download_lib "$SCRIPTPATH"/trt8.5.2 "$TRT8_5_2"
+download_lib "$SCRIPTPATH"/trt8.6.2 "$TRT8_6_2"

tflite/linux-jetson-nano/libeitrt.h

@@ -1,23 +1,35 @@
-/* Edge Impulse inferencing library
- * Copyright (c) 2020 EdgeImpulse Inc.
+/* The Clear BSD License
  *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
+ * Copyright (c) 2025 EdgeImpulse Inc.
+ * All rights reserved.
  *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted (subject to the limitations in the disclaimer
+ * below) provided that the following conditions are met:
  *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
+ *   * Redistributions of source code must retain the above copyright notice,
+ *   this list of conditions and the following disclaimer.
+ *
+ *   * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ *   * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from this
+ *   software without specific prior written permission.
+ *
+ * NO EXPRESS OR IMPLIED LICENSES TO ANY PARTY'S PATENT RIGHTS ARE GRANTED BY
+ * THIS LICENSE. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ * CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+ * PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
  */
 
 #pragma once
@@ -31,32 +43,163 @@ namespace libeitrt
 {
 
 /**
- * @brief Creates and initializes an inference engine for TensorRT.
- * If the engine has already been created from the provided file path, then
- * the engine is loaded from disk.
- * 
- * The engine is then persisted via the EiTrt object until it is deleted, 
- * to provide for fastest inference with lowest overhead
+ * @brief Creates and initializes a context for building and running TensorRT models.
+ *
+ * The models genenerated (or managed) from this context is then persisted via the EiTrt
+ * object until it is deleted,  to provide for fastest inference with lowest
+ * overhead.
  * 
  * WARNING: This function leaks..the handle can not be deleted b/c of forward declaration
  * The fix for this is to define an interface (virtual class) that has a virtual destructor
  * And also the infer function (although this way is more C friendly!)
  * My bad...should have done that from get go.
  * 
+ * @param debug enable debug if true, disable otherwise.
+ * @return std::unique_ptr<EiTrt> EiTrt handle.  Contained ptr is NULL if error
+ */
+EiTrt* create_EiTrt(bool debug);
+
+/**
+ * @brief Builds and initializes an inference engine for TensorRT.
+ * If the engine has already been created from the provided file path, then
+ * the engine is loaded from disk.
+ *
+ * The engine is then persisted via the EiTrt object until it is deleted,
+ * to provide for fastest inference with lowest overhead
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id an index to associate with the model.
  * @param model_file_name Model file path.
  * Should have hash appended so that engines are regenerated when models change!
- * @return std::unique_ptr<EiTrt> EiTrt handle.  Contained ptr is NULL if error
+ * @return true if building (or loading) the TensorRT model was successful.
+ */
+bool build(EiTrt* ei_trt_handle, int model_id, const char *model_file_name);
+
+/**
+ *  @brief Warms up the model on the GPU for given warm_up_ms ms.
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @param warm_up_ms the duration to loop and run inference.
+ * @return true if warming up the model was successful.
+ */
+bool warmUp(EiTrt* ei_trt_handle, int model_id, int warm_up_ms);
+
+/**
+ * @brief Copies input to the GPU (from CPU) for inference for model_id.
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @param input a pointer to the (float) input
+ * @param size the number of bytes to copy from the input
+ * @return true if copying the input was successful.
  */
-EiTrt* create_EiTrt(const char* model_file_name, bool debug);
+bool copyInputToDevice(EiTrt* ei_trt_handle, int model_id, float* input, int size);
 
 /**
  * @brief Perform inference
- * 
- * @param ei_trt_handle Created handle to inference engine
- * @param[in] input Input features (buffer member of ei_matrix)
- * @param[out] output Buffer to write output to 
- * @param output_size Buffer size
+ *
+ * @param ei_trt_handle EI TensorRT context.
  * @return int 0 on success, <0 otherwise
  */
-int infer(EiTrt* ei_trt_handle, float* input, float* output, int output_size);
+int infer(EiTrt* ei_trt_handle, int model_id);
+
+/**
+ * @brief Copies output to the CPU (from GPU) after inference from model_id.
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @param output a pointer to the (float) output
+ * @param size the amount of bytes to copy from the output
+ * @return true if copying the output was successful.
+ */
+bool copyOutputToHost(EiTrt* ei_trt_handle, int model_id, float* output, int size);
+
+/**
+ * @brief Configures the maximum workspace that may be allocated
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param size workspace size in bytes.
+ */
+void setMaxWorkspaceSize(EiTrt *ei_trt_handle, int size);
+
+/**
+ * @brief Returns the current configured maximum workspace size.
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @return the size of the workspace in bytes.
+ */
+int getMaxWorkspaceSize(EiTrt *ei_trt_handle);
+
+/**
+ * @brief Returns the input size (in features) of model_id.
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @return the input size (in features).
+ */
+int getInputSize(EiTrt* ei_trt_handle, int model_id);
+
+/**
+ * @brief Returns the output size (in features) of model_id.
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @return the output size (in features).
+ */
+int getOutputSize(EiTrt* ei_trt_handle, int model_id);
+
+/**
+ * @brief Returns the latest inference latency in ms for model with id
+ * (model_id) and context (ei_trt_handle).
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @return the inference time in ms.
+ **/
+uint64_t getInferenceMs(EiTrt* ei_trt_handle, int model_id);
+
+/**
+ * @brief Returns the latest inference latency in us for model with id
+ * (model_id) and context (ei_trt_handle).
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @return the inference time in us.
+ **/
+uint64_t getInferenceUs(EiTrt* ei_trt_handle, int model_id);
+
+/**
+ * @brief Returns the latest inference latency in ns for model with id
+ * (model_id) and context (ei_trt_handle).
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @param model_id a reference to the model to work on.
+ * @return the inference time in ns.
+ **/
+uint64_t getInferenceNs(EiTrt* ei_trt_handle, int model_id);
+
+/**
+ * @brief Returns the current library major version
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @return the library's major version.
+ **/
+int getMajorVersion(EiTrt *ei_trt_handle);
+
+/**
+ * @brief Returns the current library minor version
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @return the library's minor version.
+ **/
+int getMinorVersion(EiTrt *ei_trt_handle);
+
+/**
+ * @brief Returns the current library patch version
+ *
+ * @param ei_trt_handle EI TensorRT context.
+ * @return the library's patch version.
+ **/
+int getPatchVersion(EiTrt *ei_trt_handle);
 }