qt5-legatus-legacy/Lts/Plugins/ProjectionWorkstation/Common/pws_rfb_protocol.cpp

/**
 * @RDE_HEADER@
 * $Id$
 * $HeadURL$
 * $Author$
 */

#include "pws_rfb_protocol.h"

// see https://vncdotool.readthedocs.io/en/0.8.0/rfbproto.html
namespace ProjectionWorkstation
{

namespace RfbProtocol
{
namespace STC
{
static constexpr int s_bytePerPixel = 4;

enum HextileSubEncoding : uint8_t
{
  Raw                 = 0x01,
  BackgroundSpecified = 0x02,
  ForegroundSpecified = 0x04,
  AnySubrects         = 0x08,
  SubrectsColoured    = 0x10,
};

static inline int highNibble(uint8_t b)
{
  return b >> 4;
}
static inline int lowNibble(uint8_t b)
{
  return b & 0x0f;
}

static inline bool testBit(uint8_t bit, HextileSubEncoding enc)
{
  return (bit & enc) == enc;
}

static inline uint32_t swap(uint32_t in)
{
  // compiler: bswap $eax ...
  uint32_t out = 0;
  out |= ((in >> 24) & 0x000000ff);
  out |= ((in >>  8) & 0x0000ff00);
  out |= ((in <<  8) & 0x00ff0000);
  out |= ((in << 24) & 0xff000000);
  return out;
}

static inline void fillHextileRect(RfbRectangle &rect, int xOfs, int yOfs, int w, int h, uint32_t col)
{
  uint32_t *dst = reinterpret_cast<uint32_t *>(rect.pixelData.data());
  for (int y = 0; y < h; ++y)
  {
    for (int x = 0; x < w; ++x)
    {
      Q_ASSERT(x + xOfs < rect.xpos + rect.width);
      Q_ASSERT(y + yOfs < rect.ypos + rect.height);
      dst[rect.width * (y + yOfs) + (x + xOfs)] = col;
    }
  }
}

static inline void copyHextileRect(RfbRectangle &rect, int xOfs, int yOfs, int w, int h, const QByteArray &srcBa)
{
  uint32_t *dst = reinterpret_cast<uint32_t *>(rect.pixelData.data());
  const uint32_t *src = reinterpret_cast<const uint32_t *>(srcBa.data());

  for (int y = 0; y < h; ++y)
  {
    for (int x = 0; x < w; ++x)
    {
      Q_ASSERT(x + xOfs < rect.xpos + rect.width);
      Q_ASSERT(y + yOfs < rect.ypos + rect.height);
      dst[rect.width * (y + yOfs) + (x + xOfs)] = src[w * y + x];
    }
  }
}

static bool decodeHextile(RfbRectangle &rect, QDataStream &ds)
{
  uint32_t bg = 0;
  uint32_t fg = 0;

  for (int yRect = rect.ypos; yRect < rect.ypos + rect.height; yRect += 16)
  {
    for (int xRect = rect.xpos; xRect < rect.xpos + rect.width; xRect += 16)
    {
      int w = 16;
      int h = 16;
      if (rect.xpos + rect.width - xRect < 16)
        w = rect.xpos + rect.width - xRect;
      if (rect.ypos + rect.height - yRect < 16)
        h = rect.ypos + rect.height - yRect;

      const int x = xRect - rect.xpos;
      const int y = yRect - rect.ypos;

      uint8_t subencoding = 0;
      ds >> subencoding;
      if (ds.status() != QDataStream::Ok)
        return false;

      if (testBit(subencoding, HextileSubEncoding::Raw))
      {
        QByteArray buf(w * h * s_bytePerPixel, Qt::Uninitialized);
        ds.readRawData(buf.data(), buf.size());
        if (ds.status() != QDataStream::Ok)
          return false;
        copyHextileRect(rect, x, y, w, h, buf);
        continue;
      }

      if (testBit(subencoding, HextileSubEncoding::BackgroundSpecified))
      {
        ds >> bg;
        bg = swap(bg);
        if (ds.status() != QDataStream::Ok)
          return false;
      }

      fillHextileRect(rect, x, y, w, h, bg);

      if (testBit(subencoding, HextileSubEncoding::ForegroundSpecified))
      {
        ds >> fg;
        fg = swap(fg);
        if (ds.status() != QDataStream::Ok)
          return false;
      }

      if (!testBit(subencoding, HextileSubEncoding::AnySubrects))
        continue;

      uint8_t nSubrects;
      ds >> nSubrects;
      if (ds.status() != QDataStream::Ok)
        return false;

      if (testBit(subencoding, HextileSubEncoding::SubrectsColoured))
      {
        uint8_t xy = 0;
        uint8_t wh = 0;
        for (int i = 0; i < nSubrects; i++)
        {
          ds >> fg >> xy >> wh;
          fg = swap(fg);
          if (ds.status() != QDataStream::Ok)
            return false;
          int xOfs = highNibble(xy);
          int yOfs = lowNibble(xy);
          int wOfs = highNibble(wh) + 1;
          int hOfs = lowNibble(wh) + 1;
          fillHextileRect(rect, x + xOfs, y + yOfs, wOfs, hOfs, fg);
        }
      }
      else
      {
        uint8_t xy = 0;
        uint8_t wh = 0;
        for (int i = 0; i < nSubrects; i++)
        {
          ds >> xy >> wh;
          if (ds.status() != QDataStream::Ok)
            return false;
          int xOfs = highNibble(xy);
          int yOfs = lowNibble(xy);
          int wOfs = highNibble(wh) + 1;
          int hOfs = lowNibble(wh) + 1;
          fillHextileRect(rect, x + xOfs, y + yOfs, wOfs, hOfs, fg);
        }
      }
    }
  }
  return true;
}

void RfbFramebufferUpdate::fillDataFromStream(QDataStream &ds)
{
  quint8 padding;
  ds >> padding;
  if (ds.status() != QDataStream::Ok)
    return;

  quint16 numberOfRectangles;
  ds >> numberOfRectangles;
  if (ds.status() != QDataStream::Ok)
    return;

  RfbRectangle rect;
  for (int count = 0; count < numberOfRectangles; ++count)
  {
    ds >> rect.xpos >> rect.ypos >> rect.width >> rect.height >> rect.encoding;
    if (ds.status() != QDataStream::Ok)
      return;

    const int calculatedPixelData = rect.width * rect.height * s_bytePerPixel;
    if (calculatedPixelData <= 0)
      continue;
    rect.pixelData.resize(calculatedPixelData);
    if (rect.encoding == RAW_ENCODING)
    {
      ds.readRawData(rect.pixelData.data(), calculatedPixelData);
      if (ds.status() != QDataStream::Ok)
        return;
      m_rectangles.append(rect);
    }
    else if (rect.encoding == HEXTILE_ENCODING)
    {
      decodeHextile(rect, ds);
      if (ds.status() != QDataStream::Ok)
        return;
      m_rectangles.append(rect);
    }
    else if (rect.encoding == LAST_RECT_PSEUDO_ENCODING)
      break;
  }
}

} // namespace STC
} // namespace RfbProtocol
} // namespace ProjectionWorkstation