Skip to content

Instantly share code, notes, and snippets.

@nedix

nedix/gist:b1905c60743dddd52aa455e9fc3f898e

Created October 4, 2024 23:19
Show Gist options
  • Save nedix/b1905c60743dddd52aa455e9fc3f898e to your computer and use it in GitHub Desktop.
Save nedix/b1905c60743dddd52aa455e9fc3f898e to your computer and use it in GitHub Desktop.
Download ZIP
Conflicts xrdp_encoder.c
Raw
gistfile1.txt
From 95183e139112cf958a2cabd4590dda1df94b8939 Mon Sep 17 00:00:00 2001
From: NEDIX <[email protected]>
Date: Sat, 5 Oct 2024 01:17:04 +0200
Subject: [PATCH] Conflicts xrdp_encoder.c
---
xrdp/xrdp_encoder.c | 1266 ++++++++++++++++++-------------------------
1 file changed, 538 insertions(+), 728 deletions(-)
diff --git xrdp/xrdp_encoder.c xrdp/xrdp_encoder.c
index f96515bd..73bd330b 100644
--- xrdp/xrdp_encoder.c
+++ xrdp/xrdp_encoder.c
@@ -168,63 +168,78 @@ xrdp_encoder_create(struct xrdp_mm *mm)
return NULL;
}
self->mm = mm;
-
- if (mm->egfx_flags & 1)
+ self->process_enc = process_enc_egfx;
+ if (client_info->jpeg_codec_id != 0)
+ {
+ LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting jpeg codec session");
+ self->codec_id = client_info->jpeg_codec_id;
+ self->in_codec_mode = 1;
+ self->codec_quality = client_info->jpeg_prop[0];
+ client_info->capture_code = CC_SIMPLE;
+ client_info->capture_format = XRDP_a8b8g8r8;
+ self->process_enc = process_enc_jpg;
+ }
+#ifdef XRDP_X264
+ else if (mm->egfx_flags & XRDP_EGFX_H264)
{
LOG(LOG_LEVEL_INFO,
"xrdp_encoder_create: starting h264 codec session gfx");
self->in_codec_mode = 1;
- client_info->capture_code = 3;
- client_info->capture_format =
-#if AVC444
- /* XRDP_yuv444_709fr */
- (32 << 24) | (67 << 16) | (0 << 12) | (0 << 8) | (0 << 4) | 0;
-#else
- /* XRDP_nv12_709fr */
- (12 << 24) | (66 << 16) | (0 << 12) | (0 << 8) | (0 << 4) | 0;
-#endif
- self->process_enc = process_enc_h264;
+ client_info->capture_code = CC_GFX_A2;
+ client_info->capture_format = XRDP_nv12_709fr;
self->gfx = 1;
-#if defined(XRDP_VANILLA_NVIDIA_CODEC)
- self->codec_handle = xrdp_encoder_nvenc_create();
-#elif defined(XRDP_X264)
- self->codec_handle = xrdp_encoder_x264_create();
-#elif defined(XRDP_OPENH264)
- self->codec_handle = xrdp_encoder_openh264_create();
-#endif
}
+ else if (client_info->h264_codec_id != 0)
+ {
+ LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting h264 codec session");
+ self->codec_id = client_info->h264_codec_id;
+ self->in_codec_mode = 1;
+ client_info->capture_code = CC_SUF_A2;
+ client_info->capture_format = XRDP_nv12;
+ self->process_enc = process_enc_h264;
+ }
+#endif
#ifdef XRDP_RFXCODEC
- else if (mm->egfx_flags & 2)
+ else if (mm->egfx_flags & XRDP_EGFX_RFX_PRO)
{
LOG(LOG_LEVEL_INFO,
"xrdp_encoder_create: starting gfx rfx pro codec session");
self->in_codec_mode = 1;
- client_info->capture_code = 2;
- self->process_enc = process_enc_rfx;
+ client_info->capture_code = CC_GFX_PRO;
self->gfx = 1;
- self->quants = (const char *) g_rfx_quantization_values;
self->num_quants = 2;
self->quant_idx_y = 0;
self->quant_idx_u = 1;
self->quant_idx_v = 1;
- self->codec_handle = rfxcodec_encode_create(
- mm->wm->screen->width,
- mm->wm->screen->height,
- RFX_FORMAT_YUV,
- RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
- }
+ switch (client_info->mcs_connection_type)
+ {
+ case CONNECTION_TYPE_MODEM:
+ case CONNECTION_TYPE_BROADBAND_LOW:
+ case CONNECTION_TYPE_SATELLITE:
+ self->quants = (const char *) g_rfx_quantization_values_ulq;
+ break;
+ case CONNECTION_TYPE_BROADBAND_HIGH:
+ case CONNECTION_TYPE_WAN:
+ self->quants = (const char *) g_rfx_quantization_values_lq;
+ break;
+ case CONNECTION_TYPE_LAN:
+ case CONNECTION_TYPE_AUTODETECT: /* not implemented yet */
+ default:
+ self->quants = (const char *) g_rfx_quantization_values_std;
+
+ }
+ }
else if (client_info->rfx_codec_id != 0)
{
- LOG_DEVEL(LOG_LEVEL_INFO,
- "xrdp_encoder_create: starting rfx codec session");
+ LOG(LOG_LEVEL_INFO, "xrdp_encoder_create: starting rfx codec session");
self->codec_id = client_info->rfx_codec_id;
self->in_codec_mode = 1;
- client_info->capture_code = 2;
+ client_info->capture_code = CC_SUF_RFX;
self->process_enc = process_enc_rfx;
- self->codec_handle = rfxcodec_encode_create(mm->wm->screen->width,
- mm->wm->screen->height,
- RFX_FORMAT_YUV, 0);
+ self->codec_handle_rfx = rfxcodec_encode_create(mm->wm->screen->width,
+ mm->wm->screen->height,
+ RFX_FORMAT_YUV, 0);
}
#endif
else if (client_info->jpeg_codec_id != 0)
@@ -349,9 +364,9 @@ xrdp_encoder_create(struct xrdp_mm *mm)
void
xrdp_encoder_delete(struct xrdp_encoder *self)
{
- XRDP_ENC_DATA *enc;
- XRDP_ENC_DATA_DONE *enc_done;
- FIFO *fifo;
+ int index;
+
+ (void)index;
LOG_DEVEL(LOG_LEVEL_INFO, "xrdp_encoder_delete:");
if (self == 0)
@@ -363,76 +378,50 @@ xrdp_encoder_delete(struct xrdp_encoder *self)
return;
}
/* tell worker thread to shut down */
- g_set_wait_obj(self->xrdp_encoder_term);
- g_sleep(1000);
-
- /* todo delete specific encoder */
-
- if (self->process_enc == process_enc_jpg)
+ g_set_wait_obj(self->xrdp_encoder_term_request);
+ g_obj_wait(&self->xrdp_encoder_term_done, 1, NULL, 0, 5000);
+ if (!g_is_wait_obj_set(self->xrdp_encoder_term_done))
{
+ LOG(LOG_LEVEL_WARNING, "Encoder failed to shut down cleanly");
}
+
#ifdef XRDP_RFXCODEC
- else if (self->process_enc == process_enc_rfx)
+ for (index = 0; index < 16; index++)
{
- rfxcodec_encode_destroy(self->codec_handle);
+ if (self->codec_handle_prfx_gfx[index] != NULL)
+ {
+ rfxcodec_encode_destroy(self->codec_handle_prfx_gfx[index]);
+ }
}
-#endif
-#if defined(XRDP_VANILLA_NVIDIA_CODEC)
- else if (self->process_enc == process_enc_h264)
+ if (self->codec_handle_rfx != NULL)
{
- xrdp_encoder_nvenc_delete(self->codec_handle);
+ rfxcodec_encode_destroy(self->codec_handle_rfx);
}
-#elif defined(XRDP_X264)
- else if (self->process_enc == process_enc_h264)
+#endif
+
+#if defined(XRDP_X264)
+ for (index = 0; index < 16; index++)
{
- xrdp_encoder_x264_delete(self->codec_handle);
+ if (self->codec_handle_h264_gfx[index] != NULL)
+ {
+ xrdp_encoder_x264_delete(self->codec_handle_h264_gfx[index]);
+ }
}
-#elif defined(XRDP_OPENH264)
- else if (self->process_enc == process_enc_h264)
+ if (self->codec_handle_h264 != NULL)
{
- xrdp_encoder_openh264_delete(self->codec_handle);
+ xrdp_encoder_x264_delete(self->codec_handle_h264);
}
#endif
+
/* destroy wait objects used for signalling */
g_delete_wait_obj(self->xrdp_encoder_event_to_proc);
g_delete_wait_obj(self->xrdp_encoder_event_processed);
- g_delete_wait_obj(self->xrdp_encoder_term);
+ g_delete_wait_obj(self->xrdp_encoder_term_request);
+ g_delete_wait_obj(self->xrdp_encoder_term_done);
- /* cleanup fifo_to_proc */
- fifo = self->fifo_to_proc;
- if (fifo)
- {
- while (!fifo_is_empty(fifo))
- {
- enc = (XRDP_ENC_DATA *) fifo_remove_item(fifo);
- if (enc == NULL)
- {
- continue;
- }
- g_free(enc->drects);
- g_free(enc->crects);
- g_free(enc);
- }
- fifo_delete(fifo);
- }
-
- /* cleanup fifo_processed */
- fifo = self->fifo_processed;
- if (fifo)
- {
- while (!fifo_is_empty(fifo))
- {
- enc_done = (XRDP_ENC_DATA_DONE *) fifo_remove_item(fifo);
- if (enc_done == NULL)
- {
- continue;
- }
- g_free(enc_done->comp_pad_data1);
- g_free(enc_done->comp_pad_data2);
- g_free(enc_done);
- }
- fifo_delete(fifo);
- }
+ /* cleanup fifos */
+ fifo_delete(self->fifo_to_proc, NULL);
+ fifo_delete(self->fifo_processed, NULL);
tc_mutex_delete(self->mutex);
g_free(self);
}
@@ -453,7 +442,7 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
int count;
char *out_data;
XRDP_ENC_DATA_DONE *enc_done;
- FIFO *fifo_processed;
+ struct fifo *fifo_processed;
tbus mutex;
tbus event_processed;
@@ -462,29 +451,32 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
fifo_processed = self->fifo_processed;
mutex = self->mutex;
event_processed = self->xrdp_encoder_event_processed;
- count = enc->num_crects;
+ count = enc->u.sc.num_crects;
for (index = 0; index < count; index++)
{
- x = enc->crects[index * 4 + 0];
- y = enc->crects[index * 4 + 1];
- cx = enc->crects[index * 4 + 2];
- cy = enc->crects[index * 4 + 3];
+ x = enc->u.sc.crects[index * 4 + 0];
+ y = enc->u.sc.crects[index * 4 + 1];
+ cx = enc->u.sc.crects[index * 4 + 2];
+ cy = enc->u.sc.crects[index * 4 + 3];
if (cx < 1 || cy < 1)
{
LOG_DEVEL(LOG_LEVEL_WARNING, "process_enc_jpg: error 1");
continue;
}
- LOG_DEVEL(LOG_LEVEL_DEBUG, "process_enc_jpg: x %d y %d cx %d cy %d", x, y, cx, cy);
+ LOG_DEVEL(LOG_LEVEL_DEBUG, "process_enc_jpg: x %d y %d cx %d cy %d",
+ x, y, cx, cy);
out_data_bytes = MAX((cx + 4) * cy * 4, 8192);
- if ((out_data_bytes < 1) || (out_data_bytes > 16 * 1024 * 1024))
+ if ((out_data_bytes < 1)
+ || (out_data_bytes > OUT_DATA_BYTES_DEFAULT_SIZE))
{
LOG_DEVEL(LOG_LEVEL_ERROR, "process_enc_jpg: error 2");
return 1;
}
- out_data = g_new(char, out_data_bytes + 256 + 2);
- if (out_data == NULL)
+ out_data = (char *) g_malloc(out_data_bytes
+ + XRDP_SURCMD_PREFIX_BYTES + 2, 0);
+ if (out_data == 0)
{
LOG_DEVEL(LOG_LEVEL_ERROR, "process_enc_jpg: error 3");
return 1;
@@ -492,33 +484,27 @@ process_enc_jpg(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
out_data[256] = 0; /* header bytes */
out_data[257] = 0;
- error = libxrdp_codec_jpeg_compress(self->mm->wm->session, 0, enc->data,
- enc->width, enc->height,
- enc->width * 4, x, y, cx, cy,
+ error = libxrdp_codec_jpeg_compress(self->mm->wm->session, 0, enc->u.sc.data,
+ enc->u.sc.width, enc->u.sc.height,
+ enc->u.sc.width * 4, x, y, cx, cy,
quality,
- out_data + 256 + 2,
+ out_data
+ + XRDP_SURCMD_PREFIX_BYTES + 2,
&out_data_bytes);
if (error < 0)
{
- LOG_DEVEL(LOG_LEVEL_ERROR,
- "process_enc_jpg: jpeg error %d bytes %d",
- error, out_data_bytes);
+ LOG_DEVEL(LOG_LEVEL_ERROR, "process_enc_jpg: jpeg error %d "
+ "bytes %d", error, out_data_bytes);
g_free(out_data);
return 1;
}
LOG_DEVEL(LOG_LEVEL_WARNING,
"jpeg error %d bytes %d", error, out_data_bytes);
- enc_done = g_new0(XRDP_ENC_DATA_DONE, 1);
- if (enc_done == NULL)
- {
- LOG(LOG_LEVEL_INFO, "process_enc_jpg: error 3");
- return 1;
- }
-
- enc_done->pad_bytes1 = 256;
- enc_done->comp_bytes1 = out_data_bytes + 2;
- enc_done->comp_pad_data1 = out_data;
-
+ enc_done = (XRDP_ENC_DATA_DONE *)
+ g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1);
+ enc_done->comp_bytes = out_data_bytes + 2;
+ enc_done->pad_bytes = 256;
+ enc_done->comp_pad_data = out_data;
enc_done->enc = enc;
enc_done->last = index == (enc->u.sc.num_crects - 1);
enc_done->x = x;
@@ -624,15 +610,23 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
}
out_data_bytes = self->max_compressed_bytes;
- tiles_written = rfxcodec_encode(self->codec_handle,
- out_data + XRDP_SURCMD_PREFIX_BYTES,
- &out_data_bytes, enc->data,
- enc->width, enc->height,
- enc->width * 4,
- rfxrects, enc->num_drects,
- tiles, enc->num_crects,
- self->quants, self->num_quants);
+
+ encode_flags = 0;
+ if (((int)enc->flags & KEY_FRAME_REQUESTED) && encode_passes == 0)
+ {
+ encode_flags = RFX_FLAGS_PRO_KEY;
+ }
+ tiles_written = rfxcodec_encode_ex(self->codec_handle_rfx,
+ out_data + XRDP_SURCMD_PREFIX_BYTES,
+ &out_data_bytes, enc->u.sc.data,
+ enc->u.sc.width, enc->u.sc.height,
+ ((enc->u.sc.width + 63) & ~63) * 4,
+ rfxrects, enc->u.sc.num_drects,
+ tiles, enc->u.sc.num_crects,
+ self->quants, self->num_quants,
+ encode_flags);
}
+ ++encode_passes;
}
LOG_DEVEL(LOG_LEVEL_DEBUG,
@@ -680,687 +674,505 @@ process_enc_rfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
/* signal completion for main thread */
g_set_wait_obj(event_processed);
- }
- while (!finished);
-
return 0;
}
#endif
-#define SAVE_VIDEO 0
-
-#if SAVE_VIDEO
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
+#if defined(XRDP_X264)
-static int n_save_data(const char *data, int data_size, int width, int height)
+/*****************************************************************************/
+static int
+out_RFX_AVC420_METABLOCK(struct xrdp_egfx_rect *dst_rect,
+ struct stream *s,
+ struct xrdp_egfx_rect *rects,
+ int num_rects)
{
- int fd;
- struct _header
- {
- char tag[4];
- int width;
- int height;
- int bytes_follow;
- } header;
-
- fd = open("/home/christopher/video.bin", O_RDWR | O_CREAT, S_IRUSR | S_IWUSR);
- lseek(fd, 0, SEEK_END);
- header.tag[0] = 'B';
- header.tag[1] = 'E';
- header.tag[2] = 'E';
- header.tag[3] = 'F';
- header.width = width;
- header.height = height;
- header.bytes_follow = data_size;
- if (write(fd, &header, 16) != 16)
- {
- g_printf("save_data: write failed\n");
- }
+ struct xrdp_region *reg;
+ struct xrdp_rect rect;
+ int index;
+ int count;
- if (write(fd, data, data_size) != data_size)
- {
- g_printf("save_data: write failed\n");
- }
- close(fd);
+ /* RFX_AVC420_METABLOCK */
+ s_push_layer(s, iso_hdr, 4); /* numRegionRects, set later */
+ reg = xrdp_region_create(NULL);
+ if (reg == NULL)
+ {
+ return 1;
+ }
+ for (index = 0; index < num_rects; index++)
+ {
+ rect.left = MAX(0, rects[index].x1 - dst_rect->x1 - 1);
+ rect.top = MAX(0, rects[index].y1 - dst_rect->y1 - 1);
+ rect.right = MIN(dst_rect->x2 - dst_rect->x1,
+ rects[index].x2 - dst_rect->x1 + 1);
+ rect.bottom = MIN(dst_rect->y2 - dst_rect->y1,
+ rects[index].y2 - dst_rect->y1 + 1);
+ xrdp_region_add_rect(reg, &rect);
+ }
+ index = 0;
+ while (xrdp_region_get_rect(reg, index, &rect) == 0)
+ {
+ out_uint16_le(s, rect.left);
+ out_uint16_le(s, rect.top);
+ out_uint16_le(s, rect.right);
+ out_uint16_le(s, rect.bottom);
+ index++;
+ }
+ xrdp_region_delete(reg);
+ count = index;
+ while (index > 0)
+ {
+ out_uint8(s, 23); /* qp */
+ out_uint8(s, 100); /* quality level 0..100 */
+ index--;
+ }
+ s_push_layer(s, mcs_hdr, 0);
+ s_pop_layer(s, iso_hdr);
+ out_uint32_le(s, count); /* numRegionRects */
+ s_pop_layer(s, mcs_hdr);
return 0;
}
-#endif
-
-#if defined(XRDP_X264) || defined(XRDP_OPENH264) || defined(XRDP_VANILLA_NVIDIA_CODEC)
-
-#define AVC444 1
-int
-build_rfx_avc420_metablock(struct stream *s, short *rrects, int rcount, int width, int height)
+/*****************************************************************************/
+/* called from encoder thread */
+static int
+process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
{
- int index;
- int x, y, cx, cy;
- struct xrdp_enc_rect rect;
- const uint8_t qp = 22; // Default set by Microsoft.
- const uint8_t r = 0; // Required to be 0.
- const uint8_t p = 0; // Progressively encoded flag.
- int qpVal = 0;
- qpVal |= qp & 0x3F;
- qpVal |= (r & 1) << 6;
- qpVal |= (p & 1) << 7;
-
- out_uint32_le(s, rcount); /* numRegionRects */
- for (index = 0; index < rcount; index++)
- {
- int location = index * 4;
- x = rrects[location + 0];
- y = rrects[location + 1];
- cx = rrects[location + 2];
- cy = rrects[location + 3];
- rect.x = MAX(0, x);
- rect.y = MAX(0, y);
- rect.cx = MIN(x + cx, width);
- rect.cy = MIN(y + cy, height);
- /* RDPGFX_RECT16 */
- out_uint16_le(s, rect.x);
- out_uint16_le(s, rect.y);
- out_uint16_le(s, rect.cx);
- out_uint16_le(s, rect.cy);
- }
- for (index = 0; index < rcount; index++)
- {
- // 2.2.4.4.2 RDPGFX_AVC420_QUANT_QUALITY
- out_uint8(s, qpVal); /* qp */
- out_uint8(s, 100); /* quality level 0..100 (Microsoft uses 100) */
- }
- int comp_bytes_pre = 4 + rcount * 8 + rcount * 2;
- return comp_bytes_pre;
+ LOG_DEVEL(LOG_LEVEL_INFO, "process_enc_h264: dummy func");
+ return 0;
}
-static XRDP_ENC_DATA_DONE *
-build_enc_h264_avc444_yuv420_and_chroma420_stream(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
-{
- int index;
- int out_data_bytes;
-#if AVC444
- int out_data_bytes1;
-#endif
- int rcount;
- short *rrects;
- int error;
- char *out_data;
- XRDP_ENC_DATA_DONE *enc_done;
- struct stream ls;
- struct stream *s;
- int comp_bytes_pre;
-#if AVC444
- int comp_bytes_pre1;
#endif
- int enc_done_flags;
- int scr_width;
- int scr_height;
- LOG(LOG_LEVEL_DEBUG, "process_enc_x264:");
- LOG(LOG_LEVEL_DEBUG, "process_enc_x264: num_crects %d num_drects %d",
- enc->num_crects, enc->num_drects);
+/*****************************************************************************/
+static int
+gfx_send_done(struct xrdp_encoder *self, XRDP_ENC_DATA *enc,
+ int comp_bytes, int pad_bytes, char *comp_pad_data,
+ int got_frame_id, int frame_id, int is_last)
- scr_width = self->mm->wm->screen->width;
- scr_height = self->mm->wm->screen->height;
+{
+ XRDP_ENC_DATA_DONE *enc_done;
- rcount = enc->num_drects;
- rrects = enc->drects;
- if (rcount > 15)
+ enc_done = g_new0(XRDP_ENC_DATA_DONE, 1);
+ if (enc_done == NULL)
{
- rcount = enc->num_crects;
- rrects = enc->crects;
+ return 1;
}
-
- out_data_bytes = 128 * 1024 * 1024;
- index = XRDP_SURCMD_PREFIX_BYTES + 16 + 2 + enc->num_drects * 8;
- out_data = g_new0(char, out_data_bytes + index);
- if (out_data == NULL)
+ ENC_SET_BIT(enc_done->flags, ENC_DONE_FLAGS_GFX_BIT);
+ enc_done->enc = enc;
+ enc_done->last = is_last;
+ enc_done->pad_bytes = pad_bytes;
+ enc_done->comp_bytes = comp_bytes;
+ enc_done->comp_pad_data = comp_pad_data;
+ if (got_frame_id)
{
- return 0;
+ ENC_SET_BIT(enc_done->flags, ENC_DONE_FLAGS_FRAME_ID_BIT);
+ enc_done->frame_id = frame_id;
}
+ /* inform main thread done */
+ tc_mutex_lock(self->mutex);
+ fifo_add_item(self->fifo_processed, enc_done);
+ tc_mutex_unlock(self->mutex);
+ /* signal completion for main thread */
+ g_set_wait_obj(self->xrdp_encoder_event_processed);
+ return 0;
+}
+
+/*****************************************************************************/
+static struct stream *
+gfx_wiretosurface1(struct xrdp_encoder *self,
+ struct xrdp_egfx_bulk *bulk, struct stream *in_s,
+ XRDP_ENC_DATA *enc)
+{
+#ifdef XRDP_X264
+ int index;
+ int surface_id;
+ int codec_id;
+ int pixel_format;
+ int num_rects_d;
+ int num_rects_c;
+ struct stream *rv;
+ short left;
+ short top;
+ short width;
+ short height;
+ short twidth;
+ short theight;
+ int bitmap_data_length;
+ int flags;
+ struct xrdp_egfx_rect *d_rects;
+ struct xrdp_egfx_rect *c_rects;
+ struct xrdp_egfx_rect dst_rect;
+ int error;
+ struct stream ls;
+ struct stream *s;
+ short *crects;
+ struct xrdp_enc_gfx_cmd *enc_gfx_cmd = &(enc->u.gfx);
+ int mon_index;
s = &ls;
g_memset(s, 0, sizeof(struct stream));
- ls.data = out_data + XRDP_SURCMD_PREFIX_BYTES;
- ls.p = ls.data;
- ls.size = out_data_bytes + index;
-
-#if AVC444
- out_data_bytes1 = 0;
- comp_bytes_pre1 = 0;
-#endif
-
- if (self->gfx)
+ s->size = self->max_compressed_bytes;
+ s->data = g_new(char, s->size);
+ if (s->data == NULL)
{
-#if AVC444
- /* size of avc420EncodedBitmapstream1 */
- s_push_layer(s, mcs_hdr, 4);
-#endif
- /* RFX_AVC420_METABLOCK */
- comp_bytes_pre = build_rfx_avc420_metablock(s, rrects, rcount,
- scr_width, scr_height);
- enc_done_flags = 1;
+ return NULL;
}
- else
+ s->p = s->data;
+ if (!s_check_rem(in_s, 11))
{
- out_uint32_le(s, 0); /* flags */
- out_uint32_le(s, 0); /* session id */
- out_uint16_le(s, enc->width); /* src_width */
- out_uint16_le(s, enc->height); /* src_height */
- out_uint16_le(s, enc->width); /* dst_width */
- out_uint16_le(s, enc->height); /* dst_height */
- out_uint16_le(s, rcount);
- for (index = 0; index < rcount; index++)
- {
- out_uint16_le(s, rrects[index * 4 + 0]);
- out_uint16_le(s, rrects[index * 4 + 1]);
- out_uint16_le(s, rrects[index * 4 + 2]);
- out_uint16_le(s, rrects[index * 4 + 3]);
- }
- s_push_layer(s, iso_hdr, 4);
- comp_bytes_pre = 4 + 4 + 2 + 2 + 2 + 2 + 2 + rcount * 8 + 4;
- enc_done_flags = 0;
+ g_free(s->data);
+ return NULL;
}
- error = 0;
- if (enc->flags & 1)
+ in_uint16_le(in_s, surface_id);
+ in_uint16_le(in_s, codec_id);
+ in_uint8(in_s, pixel_format);
+ in_uint32_le(in_s, flags);
+ mon_index = (flags >> 28) & 0xF;
+ in_uint16_le(in_s, num_rects_d);
+ if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) ||
+ (!s_check_rem(in_s, num_rects_d * 8)))
{
- /* already compressed */
- uint8_t *ud = (uint8_t *) (enc->data);
- int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24);
- if ((cbytes < 1) || (cbytes > out_data_bytes))
- {
- LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes);
- g_free(out_data);
- return 0;
- }
- LOG(LOG_LEVEL_DEBUG,
- "process_enc_h264: already compressed and size is %d", cbytes);
- out_data_bytes = cbytes;
- g_memcpy(s->p, enc->data + 4, out_data_bytes);
+ g_free(s->data);
+ return NULL;
}
- else
+ d_rects = g_new0(struct xrdp_egfx_rect, num_rects_d);
+ if (d_rects == NULL)
{
-#if defined(XRDP_X264)
- error = xrdp_encoder_x264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data,
- s->p, &out_data_bytes);
-#elif defined(XRDP_OPENH264)
- error = xrdp_encoder_openh264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data,
- s->p, &out_data_bytes);
-#endif
+ g_free(s->data);
+ return NULL;
}
- LOG_DEVEL(LOG_LEVEL_TRACE,
- "process_enc_h264: xrdp_encoder_x264_encode rv %d "
- "out_data_bytes %d width %d height %d",
- error, out_data_bytes, enc->width, enc->height);
- if (error != 0)
+ for (index = 0; index < num_rects_d; index++)
{
- LOG_DEVEL(LOG_LEVEL_TRACE,
- "process_enc_h264: xrdp_encoder_x264_encode failed rv %d",
- error);
- g_free(out_data);
- return 0;
- }
+ in_uint16_le(in_s, left);
+ in_uint16_le(in_s, top);
+ in_uint16_le(in_s, width);
+ in_uint16_le(in_s, height);
+ d_rects[index].x1 = left;
+ d_rects[index].y1 = top;
+ d_rects[index].x2 = left + width;
+ d_rects[index].y2 = top + height;
-#if AVC444
- s->p += out_data_bytes;
- // TODO: Specify LC code here
- uint8_t LC = 0b00;
- uint32_t bitstream =
- ((uint32_t)(comp_bytes_pre + out_data_bytes) & 0x3FFFFFFFUL)
- | ((LC & 0x03UL) << 30UL);
- /* chroma 444 */
- comp_bytes_pre1 = build_rfx_avc420_metablock(s, rrects, rcount,
- scr_width, scr_height);
- out_data_bytes1 = 128 * 1024 * 1024;
-
- if (enc->flags & 1)
- {
- /* already compressed */
- uint8_t *ud = (uint8_t *) (enc->data);
- int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24);
- if ((cbytes < 1) || (cbytes > out_data_bytes))
- {
- LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes);
- g_free(out_data);
- return 0;
- }
- LOG(LOG_LEVEL_DEBUG,
- "process_enc_h264: already compressed and size is %d", cbytes);
- out_data_bytes = cbytes;
- g_memcpy(s->p, enc->data + 4, out_data_bytes);
}
- else
+ if (!s_check_rem(in_s, 2))
{
-#if defined(XRDP_X264)
- error = xrdp_encoder_x264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data
- + (enc->height * enc->width) * 3 / 2,
- s->p, &out_data_bytes1);
-#elif defined(XRDP_OPENH264)
- error = xrdp_encoder_openh264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data
- + (enc->height * enc->width) * 3 / 2,
- s->p, &out_data_bytes1);
-#endif
+ g_free(s->data);
+ g_free(d_rects);
+ return NULL;
}
- if (error != 0)
+ in_uint16_le(in_s, num_rects_c);
+ if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) ||
+ (!s_check_rem(in_s, num_rects_c * 8)))
{
- LOG_DEVEL(LOG_LEVEL_TRACE,
- "process_enc_h264: xrdp_encoder_x264_encode failed rv %d",
- error);
- g_free(out_data);
- return 0;
+ g_free(s->data);
+ g_free(d_rects);
+ return NULL;
}
- s->p += out_data_bytes1;
- s_push_layer(s, sec_hdr, 0);
- s_pop_layer(s, mcs_hdr);
- out_uint32_le(s, bitstream);
- s_pop_layer(s, sec_hdr);
-
- s->end = s->p;
-#else
- s->end = s->p + out_data_bytes;
-#endif
-
- if (s->iso_hdr != NULL)
+ c_rects = g_new0(struct xrdp_egfx_rect, num_rects_c);
+ if (c_rects == NULL)
{
- /* not used in gfx */
- s_pop_layer(s, iso_hdr);
- out_uint32_le(s, out_data_bytes);
+ g_free(s->data);
+ g_free(d_rects);
+ return NULL;
}
-
-#if SAVE_VIDEO
- n_save_data(s->p, out_data_bytes, enc->width, enc->height);
-#endif
-
- enc_done = g_new0(XRDP_ENC_DATA_DONE, 1);
- if (enc_done == NULL)
+ crects = g_new(short, num_rects_c * 4);
+ if (crects == NULL)
{
- return 0;
+ g_free(s->data);
+ g_free(c_rects);
+ g_free(d_rects);
+ return NULL;
}
-#if AVC444
- enc_done->comp_bytes1 = 4 + comp_bytes_pre
- + out_data_bytes
- + comp_bytes_pre1
- + out_data_bytes1;
-#else
- enc_done->comp_bytes = comp_bytes_pre + out_data_bytes;
-#endif
- enc_done->pad_bytes1 = XRDP_SURCMD_PREFIX_BYTES;
- enc_done->comp_pad_data1 = out_data;
- enc_done->enc = enc;
- enc_done->last = 1;
- enc_done->rect.cx = scr_width;
- enc_done->rect.cy = scr_height;
- enc_done->flags = enc_done_flags;
-
-#if 0
- g_writeln("comp_bytes_pre %d out_data_bytes %d comp_bytes_pre1 %d out_data_bytes1 %d",
- comp_bytes_pre, out_data_bytes, comp_bytes_pre1, out_data_bytes1);
- g_hexdump(enc_done->comp_pad_data + enc_done->pad_bytes, enc_done->comp_bytes);
-#endif
- return enc_done;
-}
-
-#if AVC444
-
-static XRDP_ENC_DATA_DONE *
-build_enc_h264_avc444_yuv420_stream(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
-{
- if (!self->gfx)
+ g_memcpy(crects, in_s->p, num_rects_c * 2 * 4);
+ for (index = 0; index < num_rects_c; index++)
{
- return 0;
+ in_uint16_le(in_s, left);
+ in_uint16_le(in_s, top);
+ in_uint16_le(in_s, width);
+ in_uint16_le(in_s, height);
+ c_rects[index].x1 = left;
+ c_rects[index].y1 = top;
+ c_rects[index].x2 = left + width;
+ c_rects[index].y2 = top + height;
}
-
- int index;
- int out_data_bytes;
- int rcount;
- short *rrects;
- int error;
- char *out_data;
- XRDP_ENC_DATA_DONE *enc_done;
- struct stream ls;
- struct stream *s;
- int comp_bytes_pre;
- int enc_done_flags;
- int scr_width, scr_height;
-
- LOG(LOG_LEVEL_DEBUG, "build_enc_h264_avc444_yuv420_stream:");
- LOG(LOG_LEVEL_DEBUG, "build_enc_h264_avc444_yuv420_stream: num_crects %d num_drects %d",
- enc->num_crects, enc->num_drects);
-
- scr_width = self->mm->wm->screen->width;
- scr_height = self->mm->wm->screen->height;
-
- rcount = enc->num_drects;
- rrects = enc->drects;
- if (rcount > 15)
+ if (!s_check_rem(in_s, 8))
{
- rcount = enc->num_crects;
- rrects = enc->crects;
+ g_free(s->data);
+ g_free(c_rects);
+ g_free(d_rects);
+ g_free(crects);
+ return NULL;
}
-
- out_data_bytes = 128 * 1024 * 1024;
- index = XRDP_SURCMD_PREFIX_BYTES + 16 + 2 + rcount * 8;
- out_data = g_new(char, out_data_bytes + index);
- if (out_data == NULL)
+ in_uint16_le(in_s, left);
+ in_uint16_le(in_s, top);
+ in_uint16_le(in_s, width);
+ in_uint16_le(in_s, height);
+ twidth = width;
+ theight = height;
+ dst_rect.x1 = 0;
+ dst_rect.y1 = 0;
+ dst_rect.x2 = width;
+ dst_rect.y2 = height;
+ LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface1: left %d top "
+ "%d width %d height %d mon_index %d",
+ left, top, width, height, mon_index);
+ /* RFX_AVC420_METABLOCK */
+ if (out_RFX_AVC420_METABLOCK(&dst_rect, s, d_rects, num_rects_d) != 0)
{
- return 0;
+ g_free(s->data);
+ g_free(c_rects);
+ g_free(d_rects);
+ g_free(crects);
+ LOG(LOG_LEVEL_INFO, "10");
+ return NULL;
}
- s = &ls;
- g_memset(s, 0, sizeof(struct stream));
- ls.data = out_data + XRDP_SURCMD_PREFIX_BYTES;
- ls.p = ls.data;
- ls.size = out_data_bytes + index;
+ g_free(c_rects);
+ g_free(d_rects);
- /* size of avc420EncodedBitmapstream1 */
- s_push_layer(s, mcs_hdr, 4);
- /* RFX_AVC420_METABLOCK */
- comp_bytes_pre = build_rfx_avc420_metablock(s, rrects, rcount, scr_width, scr_height);
- enc_done_flags = 1;
-
- error = 0;
- if (enc->flags & 1)
+ if (ENC_IS_BIT_SET(flags, 0))
{
/* already compressed */
- uint8_t *ud = (uint8_t *) (enc->data);
- int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24);
- if ((cbytes < 1) || (cbytes > out_data_bytes))
- {
- LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes);
- g_free(out_data);
- return 0;
- }
- LOG(LOG_LEVEL_DEBUG,
- "process_enc_h264: already compressed and size is %d", cbytes);
- out_data_bytes = cbytes;
- g_memcpy(s->p, enc->data + 4, out_data_bytes);
+ out_uint8a(s, enc_gfx_cmd->data, enc_gfx_cmd->data_bytes);
}
else
{
-#if defined(XRDP_VANILLA_NVIDIA_CODEC)
- error = xrdp_encoder_nvenc_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data,
- s->p, &out_data_bytes);
-#elif defined(XRDP_X264)
- error = xrdp_encoder_x264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data,
- s->p, &out_data_bytes);
-#elif defined(XRDP_OPENH264)
- error = xrdp_encoder_openh264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- enc->data,
- s->p, &out_data_bytes);
-#endif
- }
- LOG(LOG_LEVEL_INFO,
- "process_enc_h264: xrdp_encoder_nvenc_encode_yuv420 rv %d "
- "out_data_bytes %d width %d height %d",
- error, out_data_bytes, enc->width, enc->height);
- if (error != 0)
- {
- LOG_DEVEL(LOG_LEVEL_TRACE,
- "process_enc_h264: xrdp_encoder_x264_encode failed rv %d",
- error);
- g_free(out_data);
- return 0;
+ /* assume NV12 format */
+ if (twidth * theight * 3 / 2 > enc_gfx_cmd->data_bytes)
+ {
+ g_free(s->data);
+ g_free(crects);
+ return NULL;
+ }
+ bitmap_data_length = s_rem_out(s);
+ if (self->codec_handle_h264_gfx[mon_index] == NULL)
+ {
+ self->codec_handle_h264_gfx[mon_index] =
+ xrdp_encoder_x264_create();
+ if (self->codec_handle_h264_gfx[mon_index] == NULL)
+ {
+ g_free(s->data);
+ g_free(crects);
+ return NULL;
+ }
+ }
+ error = xrdp_encoder_x264_encode(
+ self->codec_handle_h264_gfx[mon_index], 0,
+ 0, 0,
+ width, height, twidth, theight, 0,
+ enc_gfx_cmd->data,
+ crects, num_rects_c,
+ s->p, &bitmap_data_length, NULL);
+ if (error == 0)
+ {
+ xstream_seek(s, bitmap_data_length);
+ }
+ else
+ {
+ g_free(s->data);
+ g_free(crects);
+ return NULL;
+ }
}
-
-#if SAVE_VIDEO
- n_save_data(s->p, out_data_bytes, enc->width, enc->height);
+ s_mark_end(s);
+ bitmap_data_length = (int) (s->end - s->data);
+ rv = xrdp_egfx_wire_to_surface1(bulk, surface_id,
+ codec_id,
+ pixel_format, &dst_rect,
+ s->data, bitmap_data_length);
+ g_free(s->data);
+ g_free(crects);
+ return rv;
+#else
+ (void)self;
+ (void)bulk;
+ (void)in_s;
+ (void)enc;
+ return NULL;
#endif
+}
- s->p += out_data_bytes;
-
- s_push_layer(s, sec_hdr, 0);
- s_pop_layer(s, mcs_hdr);
- // TODO: Specify LC code here
- const uint8_t LC = 0x01;
- uint32_t bitstream =
- ((uint32_t)(comp_bytes_pre + out_data_bytes) & 0x3FFFFFFFUL)
- | ((LC & 0x03UL) << 30UL);
- out_uint32_le(s, bitstream);
- s_pop_layer(s, sec_hdr);
-
- s->end = s->p;
+/*****************************************************************************/
+static struct stream *
+gfx_wiretosurface2(struct xrdp_encoder *self,
+ struct xrdp_egfx_bulk *bulk, struct stream *in_s,
+ XRDP_ENC_DATA *enc)
+{
+#ifdef XRDP_RFXCODEC
+ int index;
+ int surface_id;
+ int codec_id;
+ int codec_context_id;
+ int pixel_format;
+ int num_rects_d;
+ int num_rects_c;
+ struct stream *rv;
+ short left;
+ short top;
+ short width;
+ short height;
+ char *bitmap_data;
+ int bitmap_data_length;
+ struct rfx_tile *tiles;
+ struct rfx_rect *rfxrects;
+ int tiles_compressed;
+ int flags;
+ int total_tiles;
+ int tiles_written;
+ int mon_index;
- if (s->iso_hdr != NULL)
+ if (!s_check_rem(in_s, 15))
{
- /* not used in gfx */
- s_pop_layer(s, iso_hdr);
- out_uint32_le(s, out_data_bytes);
+ return NULL;
}
-
- enc_done = g_new0(XRDP_ENC_DATA_DONE, 1);
- if (enc_done == NULL)
+ in_uint16_le(in_s, surface_id);
+ in_uint16_le(in_s, codec_id);
+ in_uint32_le(in_s, codec_context_id);
+ in_uint8(in_s, pixel_format);
+ in_uint32_le(in_s, flags);
+ mon_index = (flags >> 28) & 0xF;
+ in_uint16_le(in_s, num_rects_d);
+ if ((num_rects_d < 1) || (num_rects_d > 16 * 1024) ||
+ (!s_check_rem(in_s, num_rects_d * 8)))
{
- return 0;
+ return NULL;
}
- enc_done->out_data_bytes1 = out_data_bytes;
- enc_done->comp_bytes1 = 4 + comp_bytes_pre + out_data_bytes;
- enc_done->pad_bytes1 = 256;
- enc_done->comp_pad_data1 = out_data;
- enc_done->enc = enc;
- enc_done->last = 1;
- enc_done->rect.cx = scr_width;
- enc_done->rect.cy = scr_height;
- enc_done->flags = enc_done_flags;
-
- return enc_done;
-}
-
-static XRDP_ENC_DATA_DONE *
-build_enc_h264_avc444_chroma420_stream(struct xrdp_encoder *self, XRDP_ENC_DATA *enc, XRDP_ENC_DATA_DONE *enc_done)
-{
- if (!self->gfx)
+ rfxrects = g_new0(struct rfx_rect, num_rects_d);
+ if (rfxrects == NULL)
{
- return 0;
+ return NULL;
}
-
- int index;
- int out_data_bytes;
- int rcount;
- short *rrects;
- int error;
- char *out_data;
- struct stream ls;
- struct stream *s;
- int comp_bytes_pre;
- int scr_width, scr_height;
-
- LOG(LOG_LEVEL_DEBUG, "build_enc_h264_avc444_chroma420_stream:");
- LOG(LOG_LEVEL_DEBUG, "build_enc_h264_avc444_chroma420_stream: num_crects %d num_drects %d",
- enc->num_crects, enc->num_drects);
-
- scr_width = self->mm->wm->screen->width;
- scr_height = self->mm->wm->screen->height;
-
- rcount = enc->num_drects;
- rrects = enc->drects;
- if (rcount > 15)
+ for (index = 0; index < num_rects_d; index++)
{
- rcount = enc->num_crects;
- rrects = enc->crects;
+ in_uint16_le(in_s, left);
+ in_uint16_le(in_s, top);
+ in_uint16_le(in_s, width);
+ in_uint16_le(in_s, height);
+ rfxrects[index].x = left;
+ rfxrects[index].y = top;
+ rfxrects[index].cx = width;
+ rfxrects[index].cy = height;
}
-
- out_data_bytes = 128 * 1024 * 1024;
- index = XRDP_SURCMD_PREFIX_BYTES + 16 + 2 + rcount * 8;
- out_data = g_new0(char, out_data_bytes + index);
- if (out_data == NULL)
+ if (!s_check_rem(in_s, 2))
{
- return 0;
+ g_free(rfxrects);
+ return NULL;
}
-
- s = &ls;
- g_memset(s, 0, sizeof(struct stream));
- ls.data = out_data + XRDP_SURCMD_PREFIX_BYTES;
- ls.p = ls.data;
- ls.size = out_data_bytes + index;
-
- /* size of avc420EncodedBitmapstream1 */
- s_push_layer(s, mcs_hdr, 4);
- /* RFX_AVC420_METABLOCK */
- comp_bytes_pre = build_rfx_avc420_metablock(s, rrects, rcount, scr_width, scr_height);
- error = 0;
- if (enc->flags & 1)
+ in_uint16_le(in_s, num_rects_c);
+ if ((num_rects_c < 1) || (num_rects_c > 16 * 1024) ||
+ (!s_check_rem(in_s, num_rects_c * 8)))
{
- /* already compressed */
- uint8_t *ud = (uint8_t *) (enc->data + enc_done->out_data_bytes1 + 4);
- int cbytes = ud[0] | (ud[1] << 8) | (ud[2] << 16) | (ud[3] << 24);
- if ((cbytes < 1) || (cbytes > out_data_bytes))
- {
- LOG(LOG_LEVEL_INFO, "process_enc_h264: bad h264 bytes %d", cbytes);
- g_free(out_data);
- return 0;
- }
- LOG(LOG_LEVEL_DEBUG,
- "process_enc_h264: already compressed and size is %d", cbytes);
- out_data_bytes = cbytes;
- g_memcpy(s->p, enc->data + enc_done->out_data_bytes1 + 8, out_data_bytes);
+ g_free(rfxrects);
+ return NULL;
}
- else
+ tiles = g_new0(struct rfx_tile, num_rects_c);
+ if (tiles == NULL)
{
- char *data_position = enc->data + (enc->height * enc->width) * 3 / 2;
-#if defined(XRDP_VANILLA_NVIDIA_CODEC)
- error = xrdp_encoder_nvenc_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- data_position,
- s->p, &out_data_bytes);
-#elif defined(XRDP_X264)
- error = xrdp_encoder_x264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- data_position,
- s->p, &out_data_bytes);
-
-#elif defined(XRDP_OPENH264)
- error = xrdp_encoder_openh264_encode(self->codec_handle, 0,
- enc->width, enc->height, 0,
- data_position,
- s->p, &out_data_bytes);
-#endif
+ g_free(rfxrects);
+ return NULL;
}
- LOG(LOG_LEVEL_INFO,
- "process_enc_h264: xrdp_encoder_nvenc_encode_chroma420 rv %d "
- "out_data_bytes %d width %d height %d",
- error, out_data_bytes, enc->width, enc->height);
- if (error != 0)
+ for (index = 0; index < num_rects_c; index++)
{
- LOG_DEVEL(LOG_LEVEL_TRACE,
- "process_enc_h264: xrdp_encoder_x264_encode failed rv %d",
- error);
- g_free(out_data);
- return 0;
+ in_uint16_le(in_s, left);
+ in_uint16_le(in_s, top);
+ in_uint16_le(in_s, width);
+ in_uint16_le(in_s, height);
+ tiles[index].x = left;
+ tiles[index].y = top;
+ tiles[index].cx = width;
+ tiles[index].cy = height;
+ tiles[index].quant_y = self->quant_idx_y;
+ tiles[index].quant_cb = self->quant_idx_u;
+ tiles[index].quant_cr = self->quant_idx_v;
}
-
-#if SAVE_VIDEO
- n_save_data(s->p, out_data_bytes, enc->width, enc->height);
-#endif
-
- s->p += out_data_bytes;
-
- s_push_layer(s, sec_hdr, 0);
- s_pop_layer(s, mcs_hdr);
-
- // TODO: Specify LC code here
- const uint8_t LC = 0x02;
- const uint32_t bitstream = ((LC & 0x03UL) << 30UL);
-
- out_uint32_le(s, bitstream);
- s_pop_layer(s, sec_hdr);
-
- s->end = s->p;
-
- if (s->iso_hdr != NULL)
+ if (!s_check_rem(in_s, 8))
{
- /* not used in gfx */
- s_pop_layer(s, iso_hdr);
- out_uint32_le(s, out_data_bytes);
+ g_free(tiles);
+ g_free(rfxrects);
+ return NULL;
}
-
- enc_done->out_data_bytes2 = out_data_bytes;
- enc_done->comp_bytes2 = 4 + comp_bytes_pre + out_data_bytes;
- enc_done->pad_bytes2 = 256;
- enc_done->comp_pad_data2 = out_data;
-
- return enc_done;
-}
-
-#endif
-
-struct xrdp_enc_rect calculateBoundingBox(short* boxes, int numBoxes)
-{
- struct xrdp_enc_rect boundingBox;
- boundingBox.x = INT16_MAX;
- boundingBox.y = INT16_MAX;
- boundingBox.cx = INT16_MIN;
- boundingBox.cy = INT16_MIN;
-
- for (int i = 0; i < numBoxes; ++i)
+ in_uint16_le(in_s, left);
+ in_uint16_le(in_s, top);
+ in_uint16_le(in_s, width);
+ in_uint16_le(in_s, height);
+ LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface2: left %d top "
+ "%d width %d height %d mon_index %d",
+ left, top, width, height, mon_index);
+ if (self->codec_handle_prfx_gfx[mon_index] == NULL)
+ {
+ self->codec_handle_prfx_gfx[mon_index] = rfxcodec_encode_create(
+ width,
+ height,
+ RFX_FORMAT_YUV,
+ RFX_FLAGS_RLGR1 | RFX_FLAGS_PRO1);
+ if (self->codec_handle_prfx_gfx[mon_index] == NULL)
+ {
+ g_free(tiles);
+ g_free(rfxrects);
+ return NULL;
+ }
+ }
+ bitmap_data_length = self->max_compressed_bytes;
+ bitmap_data = g_new(char, bitmap_data_length);
+ if (bitmap_data == NULL)
{
- int location = i * 4;
- short x = boxes[location + 0];
- short y = boxes[location + 1];
- short cx = boxes[location + 2];
- short cy = boxes[location + 3];
-
- boundingBox.x = MIN(boundingBox.x, x);
- boundingBox.y = MIN(boundingBox.y, y);
- boundingBox.cx = MAX(boundingBox.cx, cx);
- boundingBox.cy = MAX(boundingBox.cy, cy);
+ g_free(tiles);
+ g_free(rfxrects);
+ return NULL;
}
-
- return boundingBox;
-}
-
-/*****************************************************************************/
-/* called from encoder thread */
-static int
-process_enc_h264(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
-{
- FIFO *fifo_processed;
- tbus mutex;
- tbus event_processed;
-
- fifo_processed = self->fifo_processed;
- mutex = self->mutex;
- event_processed = self->xrdp_encoder_event_processed;
-
- XRDP_ENC_DATA_DONE *enc_done;
- int mode = 1;
- switch (mode) {
- case 0:
- enc_done = build_enc_h264_avc444_yuv420_and_chroma420_stream(self, enc);
+ rv = NULL;
+ tiles_written = 0;
+ total_tiles = num_rects_c;
+ for (;;)
+ {
+ tiles_compressed =
+ rfxcodec_encode(self->codec_handle_prfx_gfx[mon_index],
+ bitmap_data,
+ &bitmap_data_length,
+ enc->u.gfx.data,
+ width, height,
+ ((width + 63) & ~63) * 4,
+ rfxrects, num_rects_d,
+ tiles + tiles_written, total_tiles - tiles_written,
+ self->quants, self->num_quants);
+ if (tiles_compressed < 1)
+ {
+ break;
+ }
+ tiles_written += tiles_compressed;
+ rv = xrdp_egfx_wire_to_surface2(bulk, surface_id,
+ codec_id, codec_context_id,
+ pixel_format,
+ bitmap_data, bitmap_data_length);
+ if (rv == NULL)
+ {
+ break;
+ }
+ LOG_DEVEL(LOG_LEVEL_INFO, "gfx_wiretosurface2: "
+ "tiles_compressed %d total_tiles %d tiles_written %d",
+ tiles_compressed, total_tiles,
+ tiles_written);
+ if (tiles_written >= total_tiles)
+ {
+ /* ok, done with last tile set */
break;
- case 1:
- enc_done = build_enc_h264_avc444_yuv420_stream(self, enc);
- enc_done = build_enc_h264_avc444_chroma420_stream(self, enc, enc_done);
+ }
+ /* we have another tile set, send this one to main thread */
+ if (gfx_send_done(self, enc, (int)(rv->end - rv->data), 0,
+ rv->data, 0, 0, 0) != 0)
+ {
+ free_stream(rv);
+ rv = NULL;
break;
+ }
+ g_free(rv); /* don't call free_stream() here so s->data is valid */
+ rv = NULL;
+ bitmap_data_length = self->max_compressed_bytes;
}
-
- enc_done->rect = calculateBoundingBox(enc->drects, enc->num_drects);
-
- /* done with msg */
- /* inform main thread done */
- tc_mutex_lock(mutex);
- fifo_add_item(fifo_processed, enc_done);
- tc_mutex_unlock(mutex);
- /* signal completion for main thread */
- g_set_wait_obj(event_processed);
-
- return 0;
-}
-
+ g_free(tiles);
+ g_free(rfxrects);
+ g_free(bitmap_data);
+ return rv;
#else
(void)self;
(void)bulk;
@@ -1668,8 +1480,6 @@ process_enc_egfx(struct xrdp_encoder *self, XRDP_ENC_DATA *enc)
return 0;
}
-#endif
-
/**
* Encoder thread main loop
*****************************************************************************/
--
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment