runtime.h

// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#pragma once
#include "fastdeploy/runtime/backends/backend.h"
#include "fastdeploy/core/fd_tensor.h"
#include "fastdeploy/runtime/runtime_option.h"
#include "fastdeploy/utils/perf.h"
#ifdef ENABLE_ENCRYPTION
#include "fastdeploy/encryption/include/decrypt.h"
#endif

namespace fastdeploy {

struct FASTDEPLOY_DECL Runtime {
 public:
  bool Init(const RuntimeOption& _option);

  bool Infer(std::vector<FDTensor>& input_tensors,
             std::vector<FDTensor>* output_tensors);

  bool Infer();

  int NumInputs() { return backend_->NumInputs(); }
  int NumOutputs() { return backend_->NumOutputs(); }
  TensorInfo GetInputInfo(int index);
  TensorInfo GetOutputInfo(int index);
  std::vector<TensorInfo> GetInputInfos();
  std::vector<TensorInfo> GetOutputInfos();
  void BindInputTensor(const std::string& name, FDTensor& input);

  void BindOutputTensor(const std::string& name, FDTensor& output);

  FDTensor* GetOutputTensor(const std::string& name);

  Runtime* Clone(void* stream = nullptr, int device_id = -1);

  void ReleaseModelMemoryBuffer();

  RuntimeOption option;

  bool Compile(std::vector<std::vector<FDTensor>>& prewarm_tensors);
  double GetProfileTime() {
    return backend_->benchmark_result_.time_of_runtime;
  }

 private:
  void CreateOrtBackend();
  void CreatePaddleBackend();
  void CreateTrtBackend();
  void CreateOpenVINOBackend();
  void CreateLiteBackend();
  void CreateRKNPU2Backend();
  void CreateSophgoNPUBackend();
  void CreatePorosBackend();
  std::unique_ptr<BaseBackend> backend_;
  std::vector<FDTensor> input_tensors_;
  std::vector<FDTensor> output_tensors_;
};
}  // namespace fastdeploy