VK_NV_optical_flow

Other Extension Metadata

Last Modified Date

2022-09-26

Contributors
  • Carsten Rohde, NVIDIA
  • Vipul Parashar, NVIDIA
  • Jeff Bolz, NVIDIA
  • Eric Werness, NVIDIA

Description

Optical flow are fundamental algorithms in computer vision (CV) area. This extension allows applications to estimate 2D displacement of pixels between two frames.

This extension is designed to be used with upcoming NVIDIA Optical Flow SDK Version 5 which will be available on NVIDIA Developer webpage.

New Object Types

New Commands

New Structures

New Enums

New Bitmasks

New Enum Constants

  • VK_NV_OPTICAL_FLOW_EXTENSION_NAME
  • VK_NV_OPTICAL_FLOW_SPEC_VERSION
  • Extending VkAccessFlagBits2:
    • VK_ACCESS_2_OPTICAL_FLOW_READ_BIT_NV
    • VK_ACCESS_2_OPTICAL_FLOW_WRITE_BIT_NV
  • Extending VkFormat:
    • VK_FORMAT_R16G16_S10_5_NV
    • VK_FORMAT_R16G16_SFIXED5_NV
  • Extending VkFormatFeatureFlagBits2:
    • VK_FORMAT_FEATURE_2_OPTICAL_FLOW_COST_BIT_NV
    • VK_FORMAT_FEATURE_2_OPTICAL_FLOW_IMAGE_BIT_NV
    • VK_FORMAT_FEATURE_2_OPTICAL_FLOW_VECTOR_BIT_NV
  • Extending VkObjectType:
    • VK_OBJECT_TYPE_OPTICAL_FLOW_SESSION_NV
  • Extending VkPipelineStageFlagBits2:
    • VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV
  • Extending VkQueueFlagBits:
    • VK_QUEUE_OPTICAL_FLOW_BIT_NV
  • Extending VkStructureType:
    • VK_STRUCTURE_TYPE_OPTICAL_FLOW_EXECUTE_INFO_NV
    • VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV
    • VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_PROPERTIES_NV
    • VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_INFO_NV
    • VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_PRIVATE_DATA_INFO_NV
    • VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPTICAL_FLOW_FEATURES_NV
    • VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPTICAL_FLOW_PROPERTIES_NV

Examples

// Example querying available input formats
VkOpticalFlowImageFormatInfoNV ofFormatInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_INFO_NV };
ofFormatInfo.usage = VK_OPTICAL_FLOW_USAGE_INPUT_BIT_NV;

uint32_t count = 0;
vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, NULL);
VkOpticalFlowImageFormatPropertiesNV* fmt = new VkOpticalFlowImageFormatPropertiesNV[count];
memset(fmt, 0, count  * sizeof(VkOpticalFlowImageFormatPropertiesNV));
for (uint32_t i = 0; i < count; i++) {
    fmt[i].sType = VK_STRUCTURE_TYPE_OPTICAL_FLOW_IMAGE_FORMAT_PROPERTIES_NV;
}
vkGetPhysicalDeviceOpticalFlowImageFormatsNV(physicalDevice, &ofFormatInfo, &count, fmt);

// Pick one of the available formats
VkFormat inputFormat = fmt[0].format;

// Check feature support for optimal tiling
VkFormatProperties3 formatProperties3 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3 };
VkFormatProperties2 formatProperties2 = { VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2, &formatProperties3 };
vkGetPhysicalDeviceFormatProperties2(physicalDevice, inputFormat, &formatProperties2);
if (!(formatProperties3.optimalTilingFeatures & VK_FORMAT_FEATURE_2_OPTICAL_FLOW_IMAGE_BIT_NV)) {
    return false;
}

// Check support for image creation parameters
VkPhysicalDeviceImageFormatInfo2 imageFormatInfo2 = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2, &ofFormatInfo };
imageFormatInfo2.format = inputFormat;
imageFormatInfo2.type = VK_IMAGE_TYPE_2D;
imageFormatInfo2.tiling = VK_IMAGE_TILING_OPTIMAL;
imageFormatInfo2.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;

VkImageFormatProperties2 imageFormatProperties2 = { VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2 };
if (vkGetPhysicalDeviceImageFormatProperties2(physicalDevice, &imageFormatInfo2, &imageFormatProperties2) != VK_SUCCESS) {
    return false;
}

VkImageCreateInfo imageCreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO, &ofFormatInfo };
imageCreateInfo.imageType = VK_IMAGE_TYPE_2D;
imageCreateInfo.format = inputFormat;
imageCreateInfo.extent = { width, height, (uint32_t)1};
imageCreateInfo.mipLevels = 1;
imageCreateInfo.arrayLayers = 1;
imageCreateInfo.samples = VK_SAMPLE_COUNT_1_BIT;
imageCreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT;;
imageCreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL;

vkCreateImage(device, &imageCreateInfo, NULL, &input);
"allocate memory, bind image, create view"

"do the same for reference and output"

// Create optical flow session
VkOpticalFlowSessionCreateInfoNV sessionCreateInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_SESSION_CREATE_INFO_NV };
sessionCreateInfo.width = width;
sessionCreateInfo.height = height;
sessionCreateInfo.imageFormat = inputFormat;
sessionCreateInfo.flowVectorFormat = outputFormat;
sessionCreateInfo.outputGridSize = VK_OPTICAL_FLOW_GRID_SIZE_4X4_BIT_NV;
sessionCreateInfo.performanceLevel = VK_OPTICAL_FLOW_PERFORMANCE_LEVEL_SLOW_NV;
VkOpticalFlowSessionNV session;
vkCreateOpticalFlowSessionNV(device, &sessionCreateInfo, NULL, &session);

"allocate command buffer"

"transfer images to VK_PIPELINE_STAGE_2_OPTICAL_FLOW_BIT_NV"
"transfer input images to VK_ACCESS_2_OPTICAL_FLOW_READ_BIT_NV and output image to VK_ACCESS_2_OPTICAL_FLOW_WRITE_BIT_NV"

vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_INPUT_NV, inputView, VK_IMAGE_LAYOUT_GENERAL);
vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_REFERENCE_NV, refView, VK_IMAGE_LAYOUT_GENERAL);
vkBindOpticalFlowSessionImageNV(device, session, VK_OPTICAL_FLOW_SESSION_BINDING_POINT_FLOW_VECTOR_NV, outputView, VK_IMAGE_LAYOUT_GENERAL);

VkOpticalFlowExecuteInfoNV opticalFlowExecuteInfo = { VK_STRUCTURE_TYPE_OPTICAL_FLOW_EXECUTE_INFO_NV };
vkCmdOpticalFlowExecuteNV(cmd, session, &opticalFlowExecuteInfo);

"submit command buffer"

Version History

  • Revision 1, 2022-09-26 (Carsten Rohde)
    • Internal revisions