1
  2
  3
  4
  5
  6
  7
  8
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289

/* A flat C face for openh264's encoder.
 *
 * WHY THIS EXISTS. openh264's C API is not flat. `ISVCEncoder` is
 * `const ISVCEncoderVtbl*` — a pointer to a table of function pointers — so
 * calling Initialize or EncodeFrame means dereferencing the object, reading a
 * slot, and calling through it. jolt.ffi cannot do that: Chez fixes a foreign
 * procedure's types when it COMPILES it, and the target has to be a literal C
 * symbol name rather than a function pointer (see jolt/ffi.clj, "the target
 * must be a literal C symbol name"). So the vtable is walked here, in C, and
 * what jolt binds is the five plain symbols below.
 *
 * It is deliberately thin. No policy, no buffering beyond what openh264's own
 * output demands, and no decisions that belong in frq — the shim exists to
 * change a calling convention, not to be a video pipeline.
 *
 * THE ONE THING IT DOES DO is flatten the output. openh264 hands back an
 * SFrameBSInfo describing up to MAX_LAYER_NUM_OF_FRAME layers, each with its
 * own NAL count and a shared bitstream buffer. A caller wanting one Annex B
 * frame has to walk that; doing it in jolt would mean reading nested C structs
 * whose layout is openh264's business. It is copied into one contiguous
 * buffer owned by the encoder handle and handed over as a borrowed span,
 * valid until the next encode — which is exactly the contract frq.av already
 * has for a video frame.
 */
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include <wels/codec_api.h>
#include <wels/codec_app_def.h>

typedef struct {
  ISVCEncoder *enc;
  unsigned char *out;   /* flattened Annex B, grown as needed */
  size_t out_cap;
  int width, height;
} frq_h264;

/* Answers 0 on success, or openh264's own non-zero return. */
int frq_h264_open(int width, int height, int fps, int bitrate, void **handle) {
  ISVCEncoder *enc = NULL;
  frq_h264 *h;
  SEncParamBase p;
  int rc;

  *handle = NULL;
  rc = WelsCreateSVCEncoder(&enc);
  if (rc != 0 || enc == NULL) return rc ? rc : -1;

  memset(&p, 0, sizeof(p));
  p.iUsageType     = CAMERA_VIDEO_REAL_TIME;
  p.iPicWidth      = width;
  p.iPicHeight     = height;
  p.iTargetBitrate = bitrate;
  p.fMaxFrameRate  = (float)fps;

  rc = (*enc)->Initialize(enc, &p);
  if (rc != 0) { WelsDestroySVCEncoder(enc); return rc; }

  h = (frq_h264 *)calloc(1, sizeof(frq_h264));
  if (h == NULL) { (*enc)->Uninitialize(enc); WelsDestroySVCEncoder(enc); return -1; }
  h->enc = enc; h->width = width; h->height = height;
  *handle = h;
  return 0;
}

/* Encode one I420 frame.
 *
 * `i420` is width*height luma followed by two (width/2)*(height/2) planes.
 * On success answers 0 and sets *out / *out_len to a BORROWED span, valid
 * until the next call on this handle. *keyframe says whether it is an IDR.
 * A frame openh264 chose to skip answers 0 with *out_len == 0. */
int frq_h264_encode(void *handle, const unsigned char *i420, long long pts_us,
                    const unsigned char **out, int *out_len, int *keyframe) {
  frq_h264 *h = (frq_h264 *)handle;
  SSourcePicture pic;
  SFrameBSInfo info;
  int rc, i, j, total = 0, off = 0;

  *out = NULL; *out_len = 0; *keyframe = 0;

  memset(&pic, 0, sizeof(pic));
  pic.iPicWidth    = h->width;
  pic.iPicHeight   = h->height;
  pic.iColorFormat = videoFormatI420;
  pic.iStride[0]   = h->width;
  pic.iStride[1]   = h->width / 2;
  pic.iStride[2]   = h->width / 2;
  pic.pData[0]     = (unsigned char *)i420;
  pic.pData[1]     = pic.pData[0] + h->width * h->height;
  pic.pData[2]     = pic.pData[1] + (h->width / 2) * (h->height / 2);
  pic.uiTimeStamp  = pts_us / 1000;   /* openh264 counts milliseconds */

  memset(&info, 0, sizeof(info));
  rc = (*h->enc)->EncodeFrame(h->enc, &pic, &info);
  if (rc != cmResultSuccess) return rc;
  if (info.eFrameType == videoFrameTypeSkip) return 0;

  for (i = 0; i < info.iLayerNum; i++)
    for (j = 0; j < info.sLayerInfo[i].iNalCount; j++)
      total += info.sLayerInfo[i].pNalLengthInByte[j];

  if ((size_t)total > h->out_cap) {
    unsigned char *grown = (unsigned char *)realloc(h->out, (size_t)total);
    if (grown == NULL) return -1;
    h->out = grown; h->out_cap = (size_t)total;
  }
  for (i = 0; i < info.iLayerNum; i++) {
    int n = 0, k;
    for (k = 0; k < info.sLayerInfo[i].iNalCount; k++)
      n += info.sLayerInfo[i].pNalLengthInByte[k];
    memcpy(h->out + off, info.sLayerInfo[i].pBsBuf, (size_t)n);
    off += n;
  }

  *out = h->out;
  *out_len = total;
  *keyframe = (info.eFrameType == videoFrameTypeIDR);
  return 0;
}

int frq_h264_force_keyframe(void *handle) {
  frq_h264 *h = (frq_h264 *)handle;
  return (*h->enc)->ForceIntraFrame(h->enc, true);
}

void frq_h264_close(void *handle) {
  frq_h264 *h = (frq_h264 *)handle;
  if (h == NULL) return;
  if (h->enc) { (*h->enc)->Uninitialize(h->enc); WelsDestroySVCEncoder(h->enc); }
  free(h->out);
  free(h);
}

/* --- decoding -------------------------------------------------------------
 *
 * Same vtable problem, same answer. ISVCDecoder is `const ISVCDecoderVtbl*`,
 * so DecodeFrameNoDelay is a function pointer and jolt cannot reach it.
 *
 * What comes out is I420 in the decoder's OWN buffers, three planes with
 * their own strides — which are not the width. A decoder pads its rows, so
 * copying `width` bytes per row from a `stride`-wide plane is the mistake
 * that produces a picture sheared diagonally, and it is why the strides are
 * carried through to the converter below rather than assumed away.
 */
#include <wels/codec_def.h>

typedef struct {
  ISVCDecoder *dec;
  unsigned char *rgba;      /* converted output, grown as needed */
  size_t rgba_cap;
} frq_h264_dec;

int frq_h264_decoder_open(void **handle) {
  ISVCDecoder *dec = NULL;
  frq_h264_dec *d;
  SDecodingParam p;
  int rc;

  *handle = NULL;
  rc = WelsCreateDecoder(&dec);
  if (rc != 0 || dec == NULL) return rc ? rc : -1;

  memset(&p, 0, sizeof(p));
  p.eEcActiveIdc = ERROR_CON_SLICE_COPY;
  p.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC;

  rc = (int)(*dec)->Initialize(dec, &p);
  if (rc != 0) { WelsDestroyDecoder(dec); return rc; }

  d = (frq_h264_dec *)calloc(1, sizeof(frq_h264_dec));
  if (d == NULL) { (*dec)->Uninitialize(dec); WelsDestroyDecoder(dec); return -1; }
  d->dec = dec;
  *handle = d;
  return 0;
}

/* Decode one Annex B frame and convert it to RGBA.
 *
 * RGBA rather than I420 because that is what the far end of this is:
 * vidya/frame-rgba! takes a tightly packed RGBA buffer, and converting here
 * means the pixels are touched once, in C, instead of crossing into jolt to
 * be rearranged. On success answers 0; *out is NULL and *w/*h are 0 when the
 * decoder has no picture yet, which is normal for the first packets. */
int frq_h264_decode_rgba(void *handle, const unsigned char *annexb, int len,
                         const unsigned char **out, int *w, int *h) {
  frq_h264_dec *d = (frq_h264_dec *)handle;
  unsigned char *planes[3] = {NULL, NULL, NULL};
  SBufferInfo info;
  DECODING_STATE st;
  int width, height, y, x, sy, su, sv;
  size_t need;

  *out = NULL; *w = 0; *h = 0;
  memset(&info, 0, sizeof(info));

  st = (*d->dec)->DecodeFrameNoDelay(d->dec, annexb, len, planes, &info);
  if (st != dsErrorFree) return (int)st;
  if (info.iBufferStatus != 1) return 0;     /* no picture this time */

  width  = info.UsrData.sSystemBuffer.iWidth;
  height = info.UsrData.sSystemBuffer.iHeight;
  sy = info.UsrData.sSystemBuffer.iStride[0];
  su = info.UsrData.sSystemBuffer.iStride[1];
  sv = su;
  if (width <= 0 || height <= 0) return 0;

  need = (size_t)width * (size_t)height * 4u;
  if (need > d->rgba_cap) {
    unsigned char *grown = (unsigned char *)realloc(d->rgba, need);
    if (grown == NULL) return -1;
    d->rgba = grown; d->rgba_cap = need;
  }

  /* BT.601 limited range, integer. Not a quality decision worth agonising
   * over here: it is what a webcam stream is tagged as, and the alternative
   * is dragging a colour-management dependency in for a video call. */
  for (y = 0; y < height; y++) {
    const unsigned char *Y = planes[0] + (size_t)y * sy;
    const unsigned char *U = planes[1] + (size_t)(y / 2) * su;
    const unsigned char *V = planes[2] + (size_t)(y / 2) * sv;
    unsigned char *dst = d->rgba + (size_t)y * width * 4;
    for (x = 0; x < width; x++) {
      int c = (int)Y[x] - 16;
      int u = (int)U[x / 2] - 128;
      int v = (int)V[x / 2] - 128;
      int r = (298 * c + 409 * v + 128) >> 8;
      int g = (298 * c - 100 * u - 208 * v + 128) >> 8;
      int b = (298 * c + 516 * u + 128) >> 8;
      dst[x * 4 + 0] = (unsigned char)(r < 0 ? 0 : r > 255 ? 255 : r);
      dst[x * 4 + 1] = (unsigned char)(g < 0 ? 0 : g > 255 ? 255 : g);
      dst[x * 4 + 2] = (unsigned char)(b < 0 ? 0 : b > 255 ? 255 : b);
      dst[x * 4 + 3] = 255;
    }
  }

  *out = d->rgba; *w = width; *h = height;
  return 0;
}

void frq_h264_decoder_close(void *handle) {
  frq_h264_dec *d = (frq_h264_dec *)handle;
  if (d == NULL) return;
  if (d->dec) { (*d->dec)->Uninitialize(d->dec); WelsDestroyDecoder(d->dec); }
  free(d->rgba);
  free(d);
}

/* --- YUYV to I420 ---------------------------------------------------------
 *
 * A webcam almost never hands you I420. YUYV (4:2:2 packed) is the format
 * every UVC device supports, and openh264 wants I420 (4:2:0 planar), so
 * something has to transpose and subsample between them. Doing it in jolt
 * would be a per-pixel loop through ffi/read and ffi/write at thirty frames
 * a second; doing it here is one pass over the row pairs.
 *
 * The chroma is AVERAGED down the row pair rather than dropped. Taking every
 * other line instead is a line cheaper and shows up as combing on anything
 * with a hard colour edge — a red shirt against a pale wall is the usual
 * way to see it.
 *
 * `src` is width*height*2 bytes; `dst` is width*height*3/2. Both even
 * dimensions, which V4L2 will have negotiated anyway.
 */
void frq_yuyv_to_i420(const unsigned char *src, unsigned char *dst,
                      int width, int height) {
  int x, y;
  unsigned char *Y = dst;
  unsigned char *U = dst + width * height;
  unsigned char *V = U + (width / 2) * (height / 2);

  for (y = 0; y < height; y++) {
    const unsigned char *row = src + (size_t)y * width * 2;
    unsigned char *yr = Y + (size_t)y * width;
    for (x = 0; x < width; x++) yr[x] = row[x * 2];
  }
  for (y = 0; y < height; y += 2) {
    const unsigned char *r0 = src + (size_t)y * width * 2;
    const unsigned char *r1 = src + (size_t)(y + 1) * width * 2;
    unsigned char *ur = U + (size_t)(y / 2) * (width / 2);
    unsigned char *vr = V + (size_t)(y / 2) * (width / 2);
    for (x = 0; x < width; x += 2) {
      /* One U and one V per two pixels per row; averaged over the pair. */
      int u = (r0[x * 2 + 1] + r1[x * 2 + 1] + 1) >> 1;
      int v = (r0[x * 2 + 3] + r1[x * 2 + 3] + 1) >> 1;
      ur[x / 2] = (unsigned char)u;
      vr[x / 2] = (unsigned char)v;
    }
  }
}