OpenVINO GenAI C++ example, building OpenVINO and OpenVINO GenAI from source

_README.md

This gist shows how to build OpenVINO and OpenVINO GenAI, create a basic GenAI application, and install that application including required DLLs in a directory.

The example runs inference on CPU. To run on GPU, change CPU to GPU in example.cpp and add the following libraries to the install(FILES) section of CmakeLists.txt: openvino_intel_gpu_plugin.dll, opencl.dll (C:\Windows\System32\opencl.dll) and cache.json (.\runtime\bin\intel64\Release\cache.json).

This code was tested on Windows 11 with Microsoft Visual Studio 2022, CMake 3.28.0-msvc1 (shipped with Visual Studio), with a Developer Command Prompt. When using PowerShell, replace setupvars.bat with setupvars.ps1. Some command line tools (like curl) may work differently with PowerShell, so using cmd.exe is recommended.

See https://github.com/openvinotoolkit/openvino.genai/tree/master/samples/cpp for more OpenVINO GenAI example code.

System requirements

• Microsoft Visual Studio 2019 or higher, version 16.3 or later
• CMake 3.23 or higher (Visual Studio 2022 version is fine)
• Git for Windows
• NSIS
• WGET

Build OpenVINO

• We build the 2024.3 release branch. The instructions are the same for the master branch, just remove the git checkout line, both in this step and in the Build OpenVINO GenAI step.
• -j8 in the --build command specifies the number of cores to use. Use WMIC cpu get numberofLogicalProcessors to see the number of cores (and set it to a smaller number than that if you want to keep using your computer during building).
• OpenVINO will be installed in C:\Users\UserName\tools\openvino. Modify the --prefix argument in the --install line to change that.
• We build a very minimal OpenVINO build, disabling among others AUTO, Python and NPU support. To enable Python, remove -DENABLE_PYTHON=OFF, to enable NPU support remove -DENABLE_INTEL_NPU=OFF etc. See [CMake options for custom compilation] for an overview of options. (https://github.com/openvinotoolkit/openvino/blob/master/docs/dev/cmake_options_for_custom_compilation.md)

git clone https://github.com/openvinotoolkit/openvino.git
cd openvino
git checkout releases/2024/3 
git submodule update --init --recursive
mkdir build && cd build
cmake -G "Visual Studio 17 2022" -DCMAKE_BUILD_TYPE=Release -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_PROXY=OFF -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=OFF -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_MULTI=OFF -DENABLE_HETERO=OFF -DENABLE_TEMPLATE=OFF -DENABLE_PYTHON=OFF -DENABLE_WHEEL=OFF -DENABLE_SAMPLES=OFF -DENABLE_INTEL_NPU=OFF -DENABLE_FASTER_BUILD=ON ..
cmake --build . --config Release --verbose -j8
cmake --install . --prefix %USERPROFILE%\tools\openvino

Build OpenVINO GenAI

• If you changed the install prefix above, change it in the commands below too.
• As above, change -j8 depending on how many cores your processor has.

git clone https://github.com/openvinotoolkit/openvino.genai.git
cd openvino.genai
git checkout releases/2024/3 
git submodule update --init --recursive
call %USERPROFILE%\tools\openvino\setupvars.bat
cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/
cmake --build ./build/ --config Release --target package -j8
cmake --install ./build/ --config Release --prefix %USERPROFILE%\tools\openvino

Build the sample application

Note: Make sure to always run %USERPROFILE%\tools\openvino\setupvars.bat before building the application. For PowerShell, use setupvars.ps1.

Put CMakeLists.txt and example.cpp from this gist in a directory and cd to that directory. Then run:

mkdir build && cd build
cmake ..
cmake --build . --config Release
cmake --install . --prefix %USERPROFILE%\tools

Run the example

Running the example code requires an OpenVINO LLM model. To export an example model, run optimum-cli export openvino -m gpt2 gpt2-ov after installing the dependencies with: pip install --upgrade --upgrade-strategy eager optimum[openvino]. These dependencies are only needed for exporting the model.

NOTE: It is recommended to use a separate command prompt for exporting the model, to prevent potential environment issues between the Python environment with OpenVINO installed, and the build environment.

cd %USERPROFILE%\tools\GenaiExample
example gpt2-ov

The directory C:\Users\USER\GenaiExample contains all the necessary files for running the sample application. You can copy the directory (including the model) to another computer and run the example there. Please note that the other computer needs to have Visual C++ Redistributable installed.

CMakeLists.txt

cmake_minimum_required(VERSION 3.10)
project(GenaiExampleProject)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED True)
find_package(OpenVINOGenAI REQUIRED)
string(REPLACE "\\" "/" USERPROFILE_FIXED $ENV{USERPROFILE})
include_directories("${USERPROFILE_FIXED}/tools/openvino/runtime/include")
link_directories("${USERPROFILE_FIXED}/tools/openvino/runtime/lib/intel64/Release")
add_executable(example example.cpp)
target_link_libraries(example PRIVATE openvino::genai)
install(TARGETS example RUNTIME DESTINATION GenaiExample)
install(FILES
        "${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/openvino.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/openvino_genai.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/openvino_tokenizers.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/core_tokenizers.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/openvino_ir_frontend.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/icuuc70.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/icudt70.dll"
# tensorflow dll is not needed when OpenVINO is built with -DENABLE_TENSORFLOW_FRONTEND=OFF
# "${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/openvino_tensorflow_frontend.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/bin/intel64/Release/openvino_intel_cpu_plugin.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/3rdparty/tbb/bin/tbb12.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/3rdparty/tbb/bin/tbbmalloc.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/3rdparty/tbb/bin/tbbmalloc_proxy.dll"
"${USERPROFILE_FIXED}/tools/openvino/runtime/3rdparty/tbb/bin/tbbbind_2_5.dll"
DESTINATION GenaiExample)
# Windows-specific preprocessor definitions or compiler flags
if(WIN32)
    add_definitions(-DWINDOWS_PLATFORM)
endif()

example.cpp

#include "openvino/genai/llm_pipeline.hpp"
#include <iostream>

int main(int argc, char* argv[]) {
   std::string model_path = argv[1];
   ov::genai::LLMPipeline pipe(model_path, "CPU");
   ov::genai::GenerationConfig config = pipe.get_generation_config();
   config.max_new_tokens = 32;
   std::cout << pipe.generate("The Sun is yellow because", config);
}