| Bind the C libraries a call actually needs 87e5be9 nandi 8d ago | 1 | /* A flat C face for openh264's encoder. |
| 2 | * |
| 3 | * WHY THIS EXISTS. openh264's C API is not flat. `ISVCEncoder` is |
| 4 | * `const ISVCEncoderVtbl*` — a pointer to a table of function pointers — so |
| 5 | * calling Initialize or EncodeFrame means dereferencing the object, reading a |
| 6 | * slot, and calling through it. jolt.ffi cannot do that: Chez fixes a foreign |
| 7 | * procedure's types when it COMPILES it, and the target has to be a literal C |
| 8 | * symbol name rather than a function pointer (see jolt/ffi.clj, "the target |
| 9 | * must be a literal C symbol name"). So the vtable is walked here, in C, and |
| 10 | * what jolt binds is the five plain symbols below. |
| 11 | * |
| 12 | * It is deliberately thin. No policy, no buffering beyond what openh264's own |
| 13 | * output demands, and no decisions that belong in frq — the shim exists to |
| 14 | * change a calling convention, not to be a video pipeline. |
| 15 | * |
| 16 | * THE ONE THING IT DOES DO is flatten the output. openh264 hands back an |
| 17 | * SFrameBSInfo describing up to MAX_LAYER_NUM_OF_FRAME layers, each with its |
| 18 | * own NAL count and a shared bitstream buffer. A caller wanting one Annex B |
| 19 | * frame has to walk that; doing it in jolt would mean reading nested C structs |
| 20 | * whose layout is openh264's business. It is copied into one contiguous |
| 21 | * buffer owned by the encoder handle and handed over as a borrowed span, |
| 22 | * valid until the next encode — which is exactly the contract frq.av already |
| 23 | * has for a video frame. |
| 24 | */ |
| 25 | #include <stdlib.h> |
| 26 | #include <string.h> |
| 27 | #include <stdint.h> |
| 28 | #include <wels/codec_api.h> |
| 29 | #include <wels/codec_app_def.h> |
| 30 | |
| 31 | typedef struct { |
| 32 | ISVCEncoder *enc; |
| 33 | unsigned char *out; /* flattened Annex B, grown as needed */ |
| 34 | size_t out_cap; |
| 35 | int width, height; |
| 36 | } frq_h264; |
| 37 | |
| 38 | /* Answers 0 on success, or openh264's own non-zero return. */ |
| 39 | int frq_h264_open(int width, int height, int fps, int bitrate, void **handle) { |
| 40 | ISVCEncoder *enc = NULL; |
| 41 | frq_h264 *h; |
| 42 | SEncParamBase p; |
| 43 | int rc; |
| 44 | |
| 45 | *handle = NULL; |
| 46 | rc = WelsCreateSVCEncoder(&enc); |
| 47 | if (rc != 0 || enc == NULL) return rc ? rc : -1; |
| 48 | |
| 49 | memset(&p, 0, sizeof(p)); |
| 50 | p.iUsageType = CAMERA_VIDEO_REAL_TIME; |
| 51 | p.iPicWidth = width; |
| 52 | p.iPicHeight = height; |
| 53 | p.iTargetBitrate = bitrate; |
| 54 | p.fMaxFrameRate = (float)fps; |
| 55 | |
| 56 | rc = (*enc)->Initialize(enc, &p); |
| 57 | if (rc != 0) { WelsDestroySVCEncoder(enc); return rc; } |
| 58 | |
| 59 | h = (frq_h264 *)calloc(1, sizeof(frq_h264)); |
| 60 | if (h == NULL) { (*enc)->Uninitialize(enc); WelsDestroySVCEncoder(enc); return -1; } |
| 61 | h->enc = enc; h->width = width; h->height = height; |
| 62 | *handle = h; |
| 63 | return 0; |
| 64 | } |
| 65 | |
| 66 | /* Encode one I420 frame. |
| 67 | * |
| 68 | * `i420` is width*height luma followed by two (width/2)*(height/2) planes. |
| 69 | * On success answers 0 and sets *out / *out_len to a BORROWED span, valid |
| 70 | * until the next call on this handle. *keyframe says whether it is an IDR. |
| 71 | * A frame openh264 chose to skip answers 0 with *out_len == 0. */ |
| 72 | int frq_h264_encode(void *handle, const unsigned char *i420, long long pts_us, |
| 73 | const unsigned char **out, int *out_len, int *keyframe) { |
| 74 | frq_h264 *h = (frq_h264 *)handle; |
| 75 | SSourcePicture pic; |
| 76 | SFrameBSInfo info; |
| 77 | int rc, i, j, total = 0, off = 0; |
| 78 | |
| 79 | *out = NULL; *out_len = 0; *keyframe = 0; |
| 80 | |
| 81 | memset(&pic, 0, sizeof(pic)); |
| 82 | pic.iPicWidth = h->width; |
| 83 | pic.iPicHeight = h->height; |
| 84 | pic.iColorFormat = videoFormatI420; |
| 85 | pic.iStride[0] = h->width; |
| 86 | pic.iStride[1] = h->width / 2; |
| 87 | pic.iStride[2] = h->width / 2; |
| 88 | pic.pData[0] = (unsigned char *)i420; |
| 89 | pic.pData[1] = pic.pData[0] + h->width * h->height; |
| 90 | pic.pData[2] = pic.pData[1] + (h->width / 2) * (h->height / 2); |
| 91 | pic.uiTimeStamp = pts_us / 1000; /* openh264 counts milliseconds */ |
| 92 | |
| 93 | memset(&info, 0, sizeof(info)); |
| 94 | rc = (*h->enc)->EncodeFrame(h->enc, &pic, &info); |
| 95 | if (rc != cmResultSuccess) return rc; |
| 96 | if (info.eFrameType == videoFrameTypeSkip) return 0; |
| 97 | |
| 98 | for (i = 0; i < info.iLayerNum; i++) |
| 99 | for (j = 0; j < info.sLayerInfo[i].iNalCount; j++) |
| 100 | total += info.sLayerInfo[i].pNalLengthInByte[j]; |
| 101 | |
| 102 | if ((size_t)total > h->out_cap) { |
| 103 | unsigned char *grown = (unsigned char *)realloc(h->out, (size_t)total); |
| 104 | if (grown == NULL) return -1; |
| 105 | h->out = grown; h->out_cap = (size_t)total; |
| 106 | } |
| 107 | for (i = 0; i < info.iLayerNum; i++) { |
| 108 | int n = 0, k; |
| 109 | for (k = 0; k < info.sLayerInfo[i].iNalCount; k++) |
| 110 | n += info.sLayerInfo[i].pNalLengthInByte[k]; |
| 111 | memcpy(h->out + off, info.sLayerInfo[i].pBsBuf, (size_t)n); |
| 112 | off += n; |
| 113 | } |
| 114 | |
| 115 | *out = h->out; |
| 116 | *out_len = total; |
| 117 | *keyframe = (info.eFrameType == videoFrameTypeIDR); |
| 118 | return 0; |
| 119 | } |
| 120 | |
| 121 | int frq_h264_force_keyframe(void *handle) { |
| 122 | frq_h264 *h = (frq_h264 *)handle; |
| 123 | return (*h->enc)->ForceIntraFrame(h->enc, true); |
| 124 | } |
| 125 | |
| 126 | void frq_h264_close(void *handle) { |
| 127 | frq_h264 *h = (frq_h264 *)handle; |
| 128 | if (h == NULL) return; |
| 129 | if (h->enc) { (*h->enc)->Uninitialize(h->enc); WelsDestroySVCEncoder(h->enc); } |
| 130 | free(h->out); |
| 131 | free(h); |
| 132 | } |
| Decode H.264 to RGBA, and list the devices av.clj offers 982c702 nandi 8d ago | 133 | |
| 134 | /* --- decoding ------------------------------------------------------------- |
| 135 | * |
| 136 | * Same vtable problem, same answer. ISVCDecoder is `const ISVCDecoderVtbl*`, |
| 137 | * so DecodeFrameNoDelay is a function pointer and jolt cannot reach it. |
| 138 | * |
| 139 | * What comes out is I420 in the decoder's OWN buffers, three planes with |
| 140 | * their own strides — which are not the width. A decoder pads its rows, so |
| 141 | * copying `width` bytes per row from a `stride`-wide plane is the mistake |
| 142 | * that produces a picture sheared diagonally, and it is why the strides are |
| 143 | * carried through to the converter below rather than assumed away. |
| 144 | */ |
| 145 | #include <wels/codec_def.h> |
| 146 | |
| 147 | typedef struct { |
| 148 | ISVCDecoder *dec; |
| 149 | unsigned char *rgba; /* converted output, grown as needed */ |
| 150 | size_t rgba_cap; |
| 151 | } frq_h264_dec; |
| 152 | |
| 153 | int frq_h264_decoder_open(void **handle) { |
| 154 | ISVCDecoder *dec = NULL; |
| 155 | frq_h264_dec *d; |
| 156 | SDecodingParam p; |
| 157 | int rc; |
| 158 | |
| 159 | *handle = NULL; |
| 160 | rc = WelsCreateDecoder(&dec); |
| 161 | if (rc != 0 || dec == NULL) return rc ? rc : -1; |
| 162 | |
| 163 | memset(&p, 0, sizeof(p)); |
| 164 | p.eEcActiveIdc = ERROR_CON_SLICE_COPY; |
| 165 | p.sVideoProperty.eVideoBsType = VIDEO_BITSTREAM_AVC; |
| 166 | |
| 167 | rc = (int)(*dec)->Initialize(dec, &p); |
| 168 | if (rc != 0) { WelsDestroyDecoder(dec); return rc; } |
| 169 | |
| 170 | d = (frq_h264_dec *)calloc(1, sizeof(frq_h264_dec)); |
| 171 | if (d == NULL) { (*dec)->Uninitialize(dec); WelsDestroyDecoder(dec); return -1; } |
| 172 | d->dec = dec; |
| 173 | *handle = d; |
| 174 | return 0; |
| 175 | } |
| 176 | |
| 177 | /* Decode one Annex B frame and convert it to RGBA. |
| 178 | * |
| 179 | * RGBA rather than I420 because that is what the far end of this is: |
| 180 | * vidya/frame-rgba! takes a tightly packed RGBA buffer, and converting here |
| 181 | * means the pixels are touched once, in C, instead of crossing into jolt to |
| 182 | * be rearranged. On success answers 0; *out is NULL and *w/*h are 0 when the |
| 183 | * decoder has no picture yet, which is normal for the first packets. */ |
| 184 | int frq_h264_decode_rgba(void *handle, const unsigned char *annexb, int len, |
| 185 | const unsigned char **out, int *w, int *h) { |
| 186 | frq_h264_dec *d = (frq_h264_dec *)handle; |
| 187 | unsigned char *planes[3] = {NULL, NULL, NULL}; |
| 188 | SBufferInfo info; |
| 189 | DECODING_STATE st; |
| 190 | int width, height, y, x, sy, su, sv; |
| 191 | size_t need; |
| 192 | |
| 193 | *out = NULL; *w = 0; *h = 0; |
| 194 | memset(&info, 0, sizeof(info)); |
| 195 | |
| 196 | st = (*d->dec)->DecodeFrameNoDelay(d->dec, annexb, len, planes, &info); |
| 197 | if (st != dsErrorFree) return (int)st; |
| 198 | if (info.iBufferStatus != 1) return 0; /* no picture this time */ |
| 199 | |
| 200 | width = info.UsrData.sSystemBuffer.iWidth; |
| 201 | height = info.UsrData.sSystemBuffer.iHeight; |
| 202 | sy = info.UsrData.sSystemBuffer.iStride[0]; |
| 203 | su = info.UsrData.sSystemBuffer.iStride[1]; |
| 204 | sv = su; |
| 205 | if (width <= 0 || height <= 0) return 0; |
| 206 | |
| 207 | need = (size_t)width * (size_t)height * 4u; |
| 208 | if (need > d->rgba_cap) { |
| 209 | unsigned char *grown = (unsigned char *)realloc(d->rgba, need); |
| 210 | if (grown == NULL) return -1; |
| 211 | d->rgba = grown; d->rgba_cap = need; |
| 212 | } |
| 213 | |
| 214 | /* BT.601 limited range, integer. Not a quality decision worth agonising |
| 215 | * over here: it is what a webcam stream is tagged as, and the alternative |
| 216 | * is dragging a colour-management dependency in for a video call. */ |
| 217 | for (y = 0; y < height; y++) { |
| 218 | const unsigned char *Y = planes[0] + (size_t)y * sy; |
| 219 | const unsigned char *U = planes[1] + (size_t)(y / 2) * su; |
| 220 | const unsigned char *V = planes[2] + (size_t)(y / 2) * sv; |
| 221 | unsigned char *dst = d->rgba + (size_t)y * width * 4; |
| 222 | for (x = 0; x < width; x++) { |
| 223 | int c = (int)Y[x] - 16; |
| 224 | int u = (int)U[x / 2] - 128; |
| 225 | int v = (int)V[x / 2] - 128; |
| 226 | int r = (298 * c + 409 * v + 128) >> 8; |
| 227 | int g = (298 * c - 100 * u - 208 * v + 128) >> 8; |
| 228 | int b = (298 * c + 516 * u + 128) >> 8; |
| 229 | dst[x * 4 + 0] = (unsigned char)(r < 0 ? 0 : r > 255 ? 255 : r); |
| 230 | dst[x * 4 + 1] = (unsigned char)(g < 0 ? 0 : g > 255 ? 255 : g); |
| 231 | dst[x * 4 + 2] = (unsigned char)(b < 0 ? 0 : b > 255 ? 255 : b); |
| 232 | dst[x * 4 + 3] = 255; |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | *out = d->rgba; *w = width; *h = height; |
| 237 | return 0; |
| 238 | } |
| 239 | |
| 240 | void frq_h264_decoder_close(void *handle) { |
| 241 | frq_h264_dec *d = (frq_h264_dec *)handle; |
| 242 | if (d == NULL) return; |
| 243 | if (d->dec) { (*d->dec)->Uninitialize(d->dec); WelsDestroyDecoder(d->dec); } |
| 244 | free(d->rgba); |
| 245 | free(d); |
| 246 | } |
| Let the plane take real devices, not only test thunks 596dc29 nandi 8d ago | 247 | |
| 248 | /* --- YUYV to I420 --------------------------------------------------------- |
| 249 | * |
| 250 | * A webcam almost never hands you I420. YUYV (4:2:2 packed) is the format |
| 251 | * every UVC device supports, and openh264 wants I420 (4:2:0 planar), so |
| 252 | * something has to transpose and subsample between them. Doing it in jolt |
| 253 | * would be a per-pixel loop through ffi/read and ffi/write at thirty frames |
| 254 | * a second; doing it here is one pass over the row pairs. |
| 255 | * |
| 256 | * The chroma is AVERAGED down the row pair rather than dropped. Taking every |
| 257 | * other line instead is a line cheaper and shows up as combing on anything |
| 258 | * with a hard colour edge — a red shirt against a pale wall is the usual |
| 259 | * way to see it. |
| 260 | * |
| 261 | * `src` is width*height*2 bytes; `dst` is width*height*3/2. Both even |
| 262 | * dimensions, which V4L2 will have negotiated anyway. |
| 263 | */ |
| 264 | void frq_yuyv_to_i420(const unsigned char *src, unsigned char *dst, |
| 265 | int width, int height) { |
| 266 | int x, y; |
| 267 | unsigned char *Y = dst; |
| 268 | unsigned char *U = dst + width * height; |
| 269 | unsigned char *V = U + (width / 2) * (height / 2); |
| 270 | |
| 271 | for (y = 0; y < height; y++) { |
| 272 | const unsigned char *row = src + (size_t)y * width * 2; |
| 273 | unsigned char *yr = Y + (size_t)y * width; |
| 274 | for (x = 0; x < width; x++) yr[x] = row[x * 2]; |
| 275 | } |
| 276 | for (y = 0; y < height; y += 2) { |
| 277 | const unsigned char *r0 = src + (size_t)y * width * 2; |
| 278 | const unsigned char *r1 = src + (size_t)(y + 1) * width * 2; |
| 279 | unsigned char *ur = U + (size_t)(y / 2) * (width / 2); |
| 280 | unsigned char *vr = V + (size_t)(y / 2) * (width / 2); |
| 281 | for (x = 0; x < width; x += 2) { |
| 282 | /* One U and one V per two pixels per row; averaged over the pair. */ |
| 283 | int u = (r0[x * 2 + 1] + r1[x * 2 + 1] + 1) >> 1; |
| 284 | int v = (r0[x * 2 + 3] + r1[x * 2 + 3] + 1) >> 1; |
| 285 | ur[x / 2] = (unsigned char)u; |
| 286 | vr[x / 2] = (unsigned char)v; |
| 287 | } |
| 288 | } |
| 289 | } |