rickub
nandi/jolt-nativepublic⑂ Fork 0
⑂ 3e8c6f0df9f14601363a99d04dcdd8d1fc39f0a8
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/jolt-native.git
git clone ssh://git@rickub.com/nandi/jolt-native.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

v4l2cam.rs · 436 lines · 14.9 KBRust Blame HistoryRaw
Lift freeq's AV media plane out of sleek 90f8b89 nandi 20d ago1//! Minimal V4L2 MMAP camera capture with a **correct `dqbuf`**.
2//!
3//! `v4l2r`'s `ioctl::dqbuf` leaves `v4l2_buffer.memory = 0`. Real UVC drivers
4//! (EMEET) tolerate it, but **v4l2loopback (OBS Virtual Camera) rejects it with
5//! `EINVAL`**, which kills the capture thread on the first frame and leaves a
6//! permanently blank self-view tile. This module does the same MMAP flow with
7//! `memory = V4L2_MEMORY_MMAP` set explicitly — proven against `/dev/video10`
8//! (OBS) with `examples/v4l2_probe.rs`.
9//!
10//! Only used for loopback/virtual devices; hardware cams stay on
11//! `rusty-capture`'s richer capturer (more pixel formats, zero-copy paths).
12
13use std::fs::{File, OpenOptions};
14use std::os::unix::io::AsRawFd;
15use std::time::Instant;
16
17use anyhow::{Context, Result};
18use iroh_live::media::format::{PixelFormat, VideoFormat, VideoFrame};
19use iroh_live::media::traits::VideoSource;
20use v4l2r::bindings;
21use v4l2r::ioctl::{self, QueryBuffer};
22use v4l2r::memory::{MemoryType, MmapHandle};
23use v4l2r::{Format, PixelFormat as V4l2PixelFormat, QueueType};
24
25const V4L2_MEMORY_MMAP_U32: u32 = 1;
26/// VIDIOC_DQBUF = _IOWR('V', 17, struct v4l2_buffer). v4l2r's dqbuf helper is
27/// unusable for loopback devices (memory field never set), so issue it directly.
28const VIDIOC_DQBUF: std::ffi::c_ulong = (3 << 30)
29 | (('V' as std::ffi::c_ulong) << 8)
30 | 17
31 | ((std::mem::size_of::<bindings::v4l2_buffer>() as std::ffi::c_ulong) << 16);
32
33/// One mmap'd capture buffer (kernel-owned, re-queued after each frame).
34struct MappedBuf {
35 ptr: *mut u8,
36 len: usize,
37}
38
39// SAFETY: the mapping is process-shared memory from the kernel; access is
40// synchronized by dqbuf/qbuf ownership (we only read while dequeued to us).
41unsafe impl Send for MappedBuf {}
42
43impl Drop for MappedBuf {
44 fn drop(&mut self) {
45 if !self.ptr.is_null() && self.len > 0 {
46 unsafe {
47 libc::munmap(self.ptr.cast(), self.len);
48 }
49 }
50 }
51}
52
53/// Read the V4L2 driver name (e.g. `"v4l2 loopback"`, `"uvcvideo"`).
54pub fn driver_name(path: &str) -> Option<String> {
55 let f = OpenOptions::new().read(true).write(true).open(path).ok()?;
56 let caps: ioctl::Capability = ioctl::querycap(&f).ok()?;
57 Some(caps.driver.clone())
58}
59
60/// True when the device is a v4l2loopback node (OBS Virtual Camera etc.).
61/// Those reject v4l2r's dqbuf (memory=0) with EINVAL — route them to
62/// [`V4l2MmapCapture`] instead of rusty-capture.
63pub fn is_loopback_device(path: &str) -> bool {
64 driver_name(path)
65 .map(|d| d.to_ascii_lowercase().contains("loopback"))
66 .unwrap_or(false)
67}
68
69/// V4L2 MMAP capture with a spec-correct dqbuf. Produces RGBA frames.
70pub struct V4l2MmapCapture {
71 device_path: String,
72 name: String,
73 width: u32,
74 height: u32,
75 fourcc: [u8; 4],
76 state: Option<CaptureState>,
77}
78
79struct CaptureState {
80 dev: File,
81 bufs: Vec<MappedBuf>,
82 started: Instant,
83}
84
85impl V4l2MmapCapture {
86 /// Open `path` and negotiate `w`×`h` (driver may adjust, as v4l2loopback
87 /// does to match the OBS output). Supports YUYV and MJPG payloads.
88 pub fn open(path: &str, w: u32, h: u32) -> Result<Self> {
89 let mut dev = OpenOptions::new()
90 .read(true)
91 .write(true)
92 .open(path)
93 .with_context(|| format!("open {path}"))?;
94 let caps: ioctl::Capability = ioctl::querycap(&dev).context("querycap")?;
95
96 // Try requested size with YUYV first (loopback native), then MJPG,
97 // then let the driver pick (0x0 keeps current).
98 let mut actual: Option<Format> = None;
Run the formatter over the tree 3e8c6f0 nandi 14d ago99 for (req_w, req_h, fourcc) in [(w, h, *b"YUYV"), (w, h, *b"MJPG"), (0, 0, *b"YUYV")] {
Lift freeq's AV media plane out of sleek 90f8b89 nandi 20d ago100 let desired = Format {
101 width: req_w,
102 height: req_h,
103 pixelformat: V4l2PixelFormat::from_fourcc(&fourcc),
104 plane_fmt: vec![],
105 };
106 match ioctl::s_fmt(&mut dev, (QueueType::VideoCapture, &desired)) {
107 Ok(f) => {
108 actual = Some(f);
109 break;
110 }
111 Err(e) => log::debug!("v4l2cam: s_fmt {fourcc:?} {req_w}x{req_h}: {e}"),
112 }
113 }
114 let actual = actual.context("no acceptable V4L2 format")?;
115 let fourcc = actual.pixelformat.to_fourcc();
116 anyhow::ensure!(
117 fourcc == *b"YUYV" || fourcc == *b"MJPG",
118 "unsupported pixel format {:?} (need YUYV or MJPG)",
119 fourcc
120 );
121 log::info!(
122 "v4l2cam: opened {path} ({} / {}) {}x{} {:?}",
123 caps.card,
124 caps.driver,
125 actual.width,
126 actual.height,
127 fourcc
128 );
129 Ok(Self {
130 device_path: path.to_string(),
131 name: caps.card.clone(),
132 width: actual.width,
133 height: actual.height,
134 fourcc,
135 state: None,
136 })
137 }
138
139 fn start_streaming(&mut self) -> Result<()> {
140 if self.state.is_some() {
141 return Ok(());
142 }
143 let mut dev = OpenOptions::new()
144 .read(true)
145 .write(true)
146 .open(&self.device_path)
147 .context("reopen for streaming")?;
148 // Re-assert the negotiated format on the fresh fd.
149 let desired = Format {
150 width: self.width,
151 height: self.height,
152 pixelformat: V4l2PixelFormat::from_fourcc(&self.fourcc),
153 plane_fmt: vec![],
154 };
155 let _: Format = ioctl::s_fmt(&mut dev, (QueueType::VideoCapture, &desired))?;
156
157 let num_bufs = ioctl::reqbufs(
158 &dev,
159 QueueType::VideoCapture,
160 MemoryType::Mmap,
161 4,
162 ioctl::MemoryConsistency::empty(),
163 )
164 .context("reqbufs")?;
165 anyhow::ensure!(num_bufs > 0, "reqbufs returned 0 buffers");
166
167 let mut bufs = Vec::with_capacity(num_bufs);
168 for i in 0..num_bufs {
169 let info: QueryBuffer = ioctl::querybuf(&dev, QueueType::VideoCapture, i)?;
170 let plane = info.planes.first().context("no plane")?;
171 // v4l2r's PlaneMapping is !Send; use libc mmap so the capturer can
172 // move to the encoder thread (VideoSource: Send).
173 let ptr = unsafe {
174 libc::mmap(
175 std::ptr::null_mut(),
176 plane.length as usize,
177 libc::PROT_READ | libc::PROT_WRITE,
178 libc::MAP_SHARED,
179 dev.as_raw_fd(),
180 plane.mem_offset as libc::off_t,
181 )
182 };
183 anyhow::ensure!(ptr != libc::MAP_FAILED, "mmap buffer {i} failed");
184 bufs.push(MappedBuf {
185 ptr: ptr.cast(),
186 len: plane.length as usize,
187 });
188
189 let mut qbuf = ioctl::QBuffer::<MmapHandle>::new(QueueType::VideoCapture, i as u32);
190 qbuf.planes.push(ioctl::QBufPlane::new(0));
191 ioctl::qbuf::<_, ()>(&dev, qbuf).context("qbuf")?;
192 }
193
194 ioctl::streamon(&dev, QueueType::VideoCapture).context("streamon")?;
195 self.state = Some(CaptureState {
196 dev,
197 bufs,
198 started: Instant::now(),
199 });
200 log::debug!("v4l2cam: streaming started on {}", self.device_path);
201 Ok(())
202 }
203
204 fn stop_streaming(&mut self) {
205 if let Some(state) = self.state.take() {
206 ioctl::streamoff(&state.dev, QueueType::VideoCapture).ok();
207 log::debug!("v4l2cam: streaming stopped on {}", self.device_path);
208 }
209 }
210
211 /// dqbuf with `memory = V4L2_MEMORY_MMAP` (the field v4l2r leaves at 0,
212 /// which v4l2loopback rejects with EINVAL). Returns buffer index + bytes.
213 fn dqbuf_mmap(dev: &File) -> Result<Option<(usize, usize)>> {
214 let mut raw: bindings::v4l2_buffer = unsafe { std::mem::zeroed() };
215 raw.type_ = QueueType::VideoCapture as u32;
216 raw.memory = V4L2_MEMORY_MMAP_U32;
217 let ret = unsafe { libc::ioctl(dev.as_raw_fd(), VIDIOC_DQBUF as _, &mut raw as *mut _) };
218 if ret == 0 {
219 return Ok(Some((raw.index as usize, raw.bytesused as usize)));
220 }
221 let e = std::io::Error::last_os_error();
222 match e.raw_os_error() {
223 Some(libc::EAGAIN) => Ok(None),
224 _ => Err(anyhow::anyhow!("dqbuf: {e}")),
225 }
226 }
227
228 fn qbuf_mmap(dev: &File, index: usize) -> Result<()> {
229 let mut qbuf = ioctl::QBuffer::<MmapHandle>::new(QueueType::VideoCapture, index as u32);
230 qbuf.planes.push(ioctl::QBufPlane::new(0));
231 ioctl::qbuf::<_, ()>(dev, qbuf).context("re-queue buffer")?;
232 Ok(())
233 }
234}
235
236impl VideoSource for V4l2MmapCapture {
237 fn name(&self) -> &str {
238 &self.name
239 }
240
241 fn format(&self) -> VideoFormat {
242 VideoFormat {
243 pixel_format: PixelFormat::Rgba,
244 dimensions: [self.width, self.height],
245 }
246 }
247
248 fn start(&mut self) -> Result<()> {
249 self.start_streaming()
250 }
251
252 fn stop(&mut self) -> Result<()> {
253 self.stop_streaming();
254 Ok(())
255 }
256
257 fn pop_frame(&mut self) -> Result<Option<VideoFrame>> {
258 let Some(state) = &self.state else {
259 return Ok(None);
260 };
261 let Some((idx, bytesused)) = Self::dqbuf_mmap(&state.dev)? else {
262 return Ok(None);
263 };
264 let ts = state.started.elapsed();
265 let buf = &state.bufs[idx];
266 let n = bytesused.min(buf.len);
267 // SAFETY: buffer is dequeued to us until we re-queue below.
268 let data: &[u8] = unsafe { std::slice::from_raw_parts(buf.ptr, n) };
269
270 let rgba = if self.fourcc == *b"YUYV" {
271 Some(yuyv_to_rgba(data, self.width, self.height))
272 } else if self.fourcc == *b"MJPG" {
273 match image::load_from_memory_with_format(data, image::ImageFormat::Jpeg) {
274 Ok(img) => {
275 let rgba = img.to_rgba8();
276 if rgba.width() == self.width && rgba.height() == self.height {
277 Some(rgba.into_raw())
278 } else {
279 log::warn!("v4l2cam: MJPG dims mismatch, dropping frame");
280 None
281 }
282 }
283 Err(e) => {
284 log::warn!("v4l2cam: MJPG decode failed: {e}");
285 None
286 }
287 }
288 } else {
289 None
290 };
291
292 Self::qbuf_mmap(&state.dev, idx)?;
293
294 let Some(rgba) = rgba else {
295 return Ok(None);
296 };
297 Ok(Some(VideoFrame::new_rgba(
298 rgba.into(),
299 self.width,
300 self.height,
301 ts,
302 )))
303 }
304}
305
306impl Drop for V4l2MmapCapture {
307 fn drop(&mut self) {
308 self.stop_streaming();
309 }
310}
311
312/// Packed YUYV 4:2:2 → RGBA8 (BT.601 limited range, same as rusty-capture).
313fn yuyv_to_rgba(data: &[u8], width: u32, height: u32) -> Vec<u8> {
314 let npix = (width as usize) * (height as usize);
315 let mut rgba = vec![255u8; npix * 4];
316 let pairs = npix / 2;
317 for p in 0..pairs {
318 let o = p * 4;
319 if o + 3 >= data.len() {
320 break;
321 }
322 let y0 = data[o] as i32;
323 let u = data[o + 1] as i32 - 128;
324 let y1 = data[o + 2] as i32;
325 let v = data[o + 3] as i32 - 128;
326 let (r0, g0, b0) = yuv_to_rgb(y0, u, v);
327 let (r1, g1, b1) = yuv_to_rgb(y1, u, v);
328 let px = p * 8;
329 rgba[px] = r0;
330 rgba[px + 1] = g0;
331 rgba[px + 2] = b0;
332 rgba[px + 4] = r1;
333 rgba[px + 5] = g1;
334 rgba[px + 6] = b1;
335 }
336 rgba
337}
338
339fn yuv_to_rgb(y: i32, u: i32, v: i32) -> (u8, u8, u8) {
340 // BT.601 studio swing: C = 1.164(Y-16)
341 let c = (298 * (y - 16) + 128) >> 8;
342 let r = (c + ((409 * v + 128) >> 8)).clamp(0, 255) as u8;
343 let g = (c - ((100 * u + 208 * v + 128) >> 8)).clamp(0, 255) as u8;
344 let b = (c + ((516 * u + 128) >> 8)).clamp(0, 255) as u8;
345 (r, g, b)
346}
347
348#[cfg(test)]
349mod tests {
350 use super::*;
351 use std::time::Duration;
352
353 #[test]
354 fn yuyv_mid_gray_is_achromatic() {
355 // Y=128 U=128 V=128 → mid gray, R=G=B≈128, alpha forced 255.
356 let data = [128u8, 128, 128, 128];
357 let rgba = yuyv_to_rgba(&data, 2, 1);
358 assert_eq!(rgba.len(), 2 * 4);
359 let (r, g, b, a) = (rgba[0], rgba[1], rgba[2], rgba[3]);
360 assert_eq!(a, 255);
361 assert!((r as i16 - g as i16).abs() <= 2, "r={r} g={g}");
362 assert!((g as i16 - b as i16).abs() <= 2, "g={g} b={b}");
363 assert!((r as i16 - 128).abs() <= 4, "expected ~128, got {r}");
364 }
365
366 #[test]
367 fn yuyv_color_pair_preserves_difference() {
368 // Two luma levels must produce different RGB (real content, not uniform).
369 let data = [60u8, 90, 200, 90];
370 let rgba = yuyv_to_rgba(&data, 2, 1);
371 assert!(rgba[0] != rgba[4], "different Y must give different R");
372 }
373
374 /// Real-device integration proof: capture one frame from the loopback
375 /// device (or SLEEK_TEST_CAMERA_ID) via THIS shipped capture path and
376 /// assert non-uniform content. Skips with a reason when unavailable.
377 #[test]
378 fn v4l2_capture_yields_nonuniform_frame() {
379 let id = std::env::var("SLEEK_TEST_CAMERA_ID")
380 .ok()
381 .filter(|s| !s.is_empty())
382 .unwrap_or_else(|| "/dev/video10".to_string());
383 let mut cam = match V4l2MmapCapture::open(&id, 640, 360) {
384 Ok(c) => c,
385 Err(e) => {
386 eprintln!("SKIP v4l2_capture id={id}: open failed: {e:#}");
387 return;
388 }
389 };
390 if let Err(e) = cam.start() {
391 eprintln!("SKIP v4l2_capture id={id}: start failed: {e:#}");
392 return;
393 }
394 let deadline = Instant::now() + Duration::from_millis(2_500);
395 let mut frame = None;
396 while Instant::now() < deadline {
397 match cam.pop_frame() {
398 Ok(Some(f)) => {
399 frame = Some(f);
400 break;
401 }
402 Ok(None) => std::thread::sleep(Duration::from_millis(20)),
403 Err(e) => {
404 let _ = cam.stop();
405 eprintln!("SKIP v4l2_capture id={id}: pop_frame: {e:#}");
406 return;
407 }
408 }
409 }
410 let _ = cam.stop();
411 let Some(f) = frame else {
412 eprintln!("SKIP v4l2_capture id={id}: no frames within 2.5s");
413 return;
414 };
415 let rgba = f.rgba_image();
416 let bytes = rgba.as_raw().as_slice();
417 let mut min = 255u8;
418 let mut max = 0u8;
419 for px in bytes.chunks_exact(4) {
Run the formatter over the tree 3e8c6f0 nandi 14d ago420 let l =
421 ((77u32 * px[0] as u32 + 150u32 * px[1] as u32 + 29u32 * px[2] as u32) >> 8) as u8;
Lift freeq's AV media plane out of sleek 90f8b89 nandi 20d ago422 min = min.min(l);
423 max = max.max(l);
424 }
425 eprintln!(
426 "v4l2_capture id={id} {}x{} luma {min}..{max} (alpha0={})",
427 f.width(),
428 f.height(),
429 bytes.get(3).copied().unwrap_or(0)
430 );
431 assert!(
432 max > min,
433 "captured frame must be non-uniform (luma {min}..{max})"
434 );
435 }
436}