[host] DXGI: initial implementation of RGB24 support

This commit breaks damage tracking and the dx12 backend and is not in
a state where it should be used by the general public.

common/include/common/KVMFR.h

@@ -28,7 +28,7 @@
 #include "types.h"
 
 #define KVMFR_MAGIC   "KVMFR---"
-#define KVMFR_VERSION 19
+#define KVMFR_VERSION 20
 
 #define KVMFR_MAX_DAMAGE_RECTS 64
 
@@ -149,6 +149,8 @@ typedef struct KVMFRFrame
   FrameType       type;               // the frame data type
   uint32_t        screenWidth;        // the client's screen width
   uint32_t        screenHeight;       // the client's screen height
+  uint32_t        dataWidth;          // the packed width of the frame data
+  uint32_t        dataHeight;         // the packed height of the frame data
   uint32_t        frameWidth;         // the frame width
   uint32_t        frameHeight;        // the frame height
   FrameRotation   rotation;           // the frame rotation

common/include/common/types.h

@@ -55,6 +55,7 @@ typedef enum FrameType
   FRAME_TYPE_RGBA      , // RGBA interleaved: R,G,B,A 32bpp
   FRAME_TYPE_RGBA10    , // RGBA interleaved: R,G,B,A 10,10,10,2 bpp
   FRAME_TYPE_RGBA16F   , // RGBA interleaved: R,G,B,A 16,16,16,16 bpp float
+  FRAME_TYPE_BGR       , // BGR  interleaved: B,G,R 24bpp
   FRAME_TYPE_MAX       , // sentinel value
 }
 FrameType;

common/src/KVMFR.c

@@ -26,5 +26,6 @@ const char * FrameTypeStr[FRAME_TYPE_MAX] =
   "FRAME_TYPE_BGRA",
   "FRAME_TYPE_RGBA",
   "FRAME_TYPE_RGBA10",
-  "FRAME_TYPE_RGBA16F"
+  "FRAME_TYPE_RGBA16F",
+  "FRAME_TYPE_BGR"
 };

host/include/interface/capture.h

@@ -48,6 +48,7 @@ typedef enum CaptureFormat
   CAPTURE_FMT_RGBA   ,
   CAPTURE_FMT_RGBA10 ,
   CAPTURE_FMT_RGBA16F,
+  CAPTURE_FMT_BGR    ,
 
   // pointer formats
   CAPTURE_FMT_COLOR ,
@@ -69,18 +70,21 @@ CaptureRotation;
 
 typedef struct CaptureFrame
 {
-  unsigned int    formatVer;
-  unsigned int    screenWidth;
-  unsigned int    screenHeight;
-  unsigned int    frameWidth;
-  unsigned int    frameHeight;
-  bool            truncated;
-  unsigned int    pitch;
-  unsigned int    stride;
-  CaptureFormat   format;
-  bool            hdr;
-  bool            hdrPQ;
-  CaptureRotation rotation;
+  unsigned        formatVer;
+  unsigned        screenWidth;  // actual screen width
+  unsigned        screenHeight; // actual screen height
+  unsigned        dataWidth;    // the width of the packed frame data
+  unsigned        dataHeight;   // the height of the packed frame data
+  unsigned        frameWidth;   // width of the frame image
+  unsigned        frameHeight;  // height of the frame image
+  unsigned        pitch;        // total width of one row of data in bytes
+  unsigned        stride;       // total width of one row of data in pixels
+  CaptureFormat   format;       // the data format of the frame
+  bool            truncated;    // true if the frame data is truncated
+  bool            hdr;          // true if the frame format is HDR
+  bool            hdrPQ;        // true if the frame format is PQ transformed
+  CaptureRotation rotation;     // output rotation of the frame
+
   uint32_t        damageRectsCount;
   FrameDamageRect damageRects[KVMFR_MAX_DAMAGE_RECTS];
 }
@@ -94,9 +98,9 @@ typedef struct CapturePointer
 
   bool          shapeUpdate;
   CaptureFormat format;
-  unsigned int  hx, hy;
-  unsigned int  width, height;
-  unsigned int  pitch;
+  unsigned      hx, hy;
+  unsigned      width, height;
+  unsigned      pitch;
 }
 CapturePointer;
 
@@ -123,6 +127,6 @@ typedef struct CaptureInterface
 
   CaptureResult (*capture   )(void);
   CaptureResult (*waitFrame )(CaptureFrame * frame, const size_t maxFrameSize);
-  CaptureResult (*getFrame  )(FrameBuffer  * frame, const unsigned int height, int frameIndex);
+  CaptureResult (*getFrame  )(FrameBuffer  * frame, int frameIndex);
 }
 CaptureInterface;

host/platform/Windows/capture/DXGI/CMakeLists.txt

@@ -10,6 +10,7 @@ add_library(capture_DXGI STATIC
   src/com_ref.c
 
   src/pp/sdrwhitelevel.c
+  src/pp/rgb24.c
 )
 
 add_definitions("-DCOBJMACROS -DINITGUID")

host/platform/Windows/capture/DXGI/src/d3d11.c

@@ -35,9 +35,7 @@ struct D3D11Backend
 
 struct D3D11TexImpl
 {
-  ID3D11Texture2D          ** gpu;
-  ID3D11Texture2D          ** cpu;
-  ID3D11ShaderResourceView ** srv;
+  ID3D11Texture2D ** cpu;
 };
 
 #define TEXIMPL(x) ((struct D3D11TexImpl *)(x).impl)
@@ -49,7 +47,6 @@ static void d3d11_free(void);
 
 static bool d3d11_create(struct DXGIInterface * intf)
 {
-  HRESULT status;
   dxgi = intf;
 
   DEBUG_ASSERT(!this);
@@ -61,32 +58,24 @@ static bool d3d11_create(struct DXGIInterface * intf)
   }
 
   this->avgMapTime = runningavg_new(10);
+  return true;
+}
 
-  D3D11_TEXTURE2D_DESC gpuTexDesc =
-  {
-    .Width              = dxgi->width,
-    .Height             = dxgi->height,
-    .MipLevels          = dxgi->downsampleLevel + 1,
-    .ArraySize          = 1,
-    .SampleDesc.Count   = 1,
-    .SampleDesc.Quality = 0,
-    .Usage              = D3D11_USAGE_DEFAULT,
-    .Format             = dxgi->dxgiFormat,
-    .BindFlags          = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE,
-    .CPUAccessFlags     = 0,
-    .MiscFlags          = D3D11_RESOURCE_MISC_GENERATE_MIPS
-  };
+static bool d3d11_configure(unsigned width, unsigned height, DXGI_FORMAT format,
+  unsigned * pitch)
+{
+  HRESULT status;
 
   D3D11_TEXTURE2D_DESC cpuTexDesc =
   {
-    .Width              = dxgi->targetWidth,
-    .Height             = dxgi->targetHeight,
+    .Width              = width,
+    .Height             = height,
     .MipLevels          = 1,
     .ArraySize          = 1,
     .SampleDesc.Count   = 1,
     .SampleDesc.Quality = 0,
     .Usage              = D3D11_USAGE_STAGING,
-    .Format             = dxgi->dxgiFormat,
+    .Format             = format,
     .BindFlags          = 0,
     .CPUAccessFlags     = D3D11_CPU_ACCESS_READ,
     .MiscFlags          = 0
@@ -111,22 +100,6 @@ static bool d3d11_create(struct DXGIInterface * intf)
       DEBUG_WINERROR("Failed to create CPU texture", status);
       goto fail;
     }
-
-    if (!dxgi->downsampleLevel)
-      continue;
-
-    status = ID3D11Device_CreateTexture2D(*dxgi->device, &gpuTexDesc, NULL,
-      (ID3D11Texture2D **)comRef_newGlobal(&teximpl->gpu));
-
-    if (FAILED(status))
-    {
-      DEBUG_WINERROR("Failed to create GPU texture", status);
-      goto fail;
-    }
-
-    ID3D11Device_CreateShaderResourceView(*dxgi->device,
-      *(ID3D11Resource **)teximpl->gpu, NULL,
-      (ID3D11ShaderResourceView **)comRef_newGlobal(&teximpl->srv));
   }
 
   // map the texture simply to get the pitch and stride
@@ -141,16 +114,13 @@ static bool d3d11_create(struct DXGIInterface * intf)
     goto fail;
   }
 
-  dxgi->pitch  = mapping.RowPitch;
-  dxgi->stride = mapping.RowPitch / dxgi->bpp;
-
   ID3D11DeviceContext_Unmap(*dxgi->deviceContext,
     *(ID3D11Resource **)TEXIMPL(dxgi->texture[0])->cpu, 0);
 
+  *pitch = mapping.RowPitch;
   return true;
 
 fail:
-  d3d11_free();
   return false;
 }
 
@@ -199,76 +169,12 @@ static void copyFrameFull(Texture * tex, ID3D11Texture2D * src)
   }
 }
 
-static void copyFrameDownsampled(Texture * tex, ID3D11Texture2D * src)
-{
-  struct D3D11TexImpl * teximpl = TEXIMPL(*tex);
-  ID3D11Texture2D * dst = *teximpl->gpu;
-
-  if (tex->texDamageCount < 0)
-    ID3D11DeviceContext_ResolveSubresource(*dxgi->deviceContext,
-      (ID3D11Resource *)dst, 0,
-      (ID3D11Resource *)src, 0,
-      dxgi->dxgiFormat);
-  else
-  {
-    for (int i = 0; i < tex->texDamageCount; ++i)
-    {
-      FrameDamageRect * rect = tex->texDamageRects + i;
-      D3D11_BOX box =
-      {
-        .left   = rect->x << dxgi->downsampleLevel,
-        .top    = rect->y << dxgi->downsampleLevel,
-        .front  = 0,
-        .back   = 1,
-        .right  = (rect->x + rect->width ) << dxgi->downsampleLevel,
-        .bottom = (rect->y + rect->height) << dxgi->downsampleLevel,
-      };
-      ID3D11DeviceContext_CopySubresourceRegion(*dxgi->deviceContext,
-        (ID3D11Resource *)dst, 0, box.left, box.top, 0,
-        (ID3D11Resource *)src, 0, &box);
-    }
-  }
-
-  ID3D11DeviceContext_GenerateMips(*dxgi->deviceContext, *teximpl->srv);
-  if (tex->texDamageCount < 0)
-    ID3D11DeviceContext_CopySubresourceRegion(*dxgi->deviceContext,
-      *(ID3D11Resource **)teximpl->cpu, 0, 0, 0, 0,
-       (ID3D11Resource * )dst          , dxgi->downsampleLevel, NULL);
-  else
-  {
-    for (int i = 0; i < tex->texDamageCount; ++i)
-    {
-      FrameDamageRect * rect = tex->texDamageRects + i;
-      D3D11_BOX box =
-      {
-        .left   = rect->x,
-        .top    = rect->y,
-        .front  = 0,
-        .back   = 1,
-        .right  = rect->x + rect->width ,
-        .bottom = rect->y + rect->height,
-      };
-
-      ID3D11DeviceContext_CopySubresourceRegion(*dxgi->deviceContext,
-        *(ID3D11Resource **)teximpl->cpu, 0, box.left, box.top, 0,
-         (ID3D11Resource * )dst          , dxgi->downsampleLevel, &box);
-    }
-  }
-}
-
 static bool d3d11_copyFrame(Texture * tex, ID3D11Texture2D * src)
 {
-  struct D3D11TexImpl * teximpl = TEXIMPL(*tex);
-
   INTERLOCKED_SECTION(dxgi->deviceContextLock,
   {
     tex->copyTime = microtime();
-
-    if (teximpl->gpu && *teximpl->gpu)
-      copyFrameDownsampled(tex, src);
-    else
-      copyFrameFull(tex, src);
-
+    copyFrameFull(tex, src);
     ID3D11DeviceContext_Flush(*dxgi->deviceContext);
   });
   return true;
@@ -339,6 +245,7 @@ struct DXGICopyBackend copyBackendD3D11 = {
   .name         = "Direct3D 11",
   .code         = "d3d11",
   .create       = d3d11_create,
+  .configure    = d3d11_configure,
   .free         = d3d11_free,
   .copyFrame    = d3d11_copyFrame,
   .mapTexture   = d3d11_mapTexture,

host/platform/Windows/capture/DXGI/src/d3d12.c

@@ -95,8 +95,8 @@ static bool d3d12_create(struct DXGIInterface * intf)
   {
     DEBUG_WARN("The D3D12 backend does not support downsampling yet");
     dxgi->downsampleLevel = 0;
-    dxgi->targetWidth     = dxgi->width;
-    dxgi->targetHeight    = dxgi->height;
+    dxgi->outputWidth     = dxgi->width;
+    dxgi->outputHeight    = dxgi->height;
   }
 
   if (dxgi->debug)