rickub
nandi/jolt-nativepublic⑂ Fork 0
⑂ 600f207e2f91fc746a6545b141494a260e410f60
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/jolt-native.git
git clone ssh://git@rickub.com/nandi/jolt-native.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

v4l2cam.rs · 439 lines · 15.0 KBRust Blame HistoryRaw
Lift freeq's AV media plane out of sleek 90f8b89 nandi 20d ago1//! Minimal V4L2 MMAP camera capture with a **correct `dqbuf`**.
2//!
3//! `v4l2r`'s `ioctl::dqbuf` leaves `v4l2_buffer.memory = 0`. Real UVC drivers
4//! (EMEET) tolerate it, but **v4l2loopback (OBS Virtual Camera) rejects it with
5//! `EINVAL`**, which kills the capture thread on the first frame and leaves a
6//! permanently blank self-view tile. This module does the same MMAP flow with
7//! `memory = V4L2_MEMORY_MMAP` set explicitly — proven against `/dev/video10`
8//! (OBS) with `examples/v4l2_probe.rs`.
9//!
10//! Only used for loopback/virtual devices; hardware cams stay on
11//! `rusty-capture`'s richer capturer (more pixel formats, zero-copy paths).
12
13use std::fs::{File, OpenOptions};
14use std::os::unix::io::AsRawFd;
15use std::time::Instant;
16
17use anyhow::{Context, Result};
18use iroh_live::media::format::{PixelFormat, VideoFormat, VideoFrame};
19use iroh_live::media::traits::VideoSource;
20use v4l2r::bindings;
21use v4l2r::ioctl::{self, QueryBuffer};
22use v4l2r::memory::{MemoryType, MmapHandle};
23use v4l2r::{Format, PixelFormat as V4l2PixelFormat, QueueType};
24
25const V4L2_MEMORY_MMAP_U32: u32 = 1;
26/// VIDIOC_DQBUF = _IOWR('V', 17, struct v4l2_buffer). v4l2r's dqbuf helper is
27/// unusable for loopback devices (memory field never set), so issue it directly.
28const VIDIOC_DQBUF: std::ffi::c_ulong = (3 << 30)
29 | (('V' as std::ffi::c_ulong) << 8)
30 | 17
31 | ((std::mem::size_of::<bindings::v4l2_buffer>() as std::ffi::c_ulong) << 16);
32
33/// One mmap'd capture buffer (kernel-owned, re-queued after each frame).
34struct MappedBuf {
35 ptr: *mut u8,
36 len: usize,
37}
38
39// SAFETY: the mapping is process-shared memory from the kernel; access is
40// synchronized by dqbuf/qbuf ownership (we only read while dequeued to us).
41unsafe impl Send for MappedBuf {}
42
43impl Drop for MappedBuf {
44 fn drop(&mut self) {
45 if !self.ptr.is_null() && self.len > 0 {
46 unsafe {
47 libc::munmap(self.ptr.cast(), self.len);
48 }
49 }
50 }
51}
52
53/// Read the V4L2 driver name (e.g. `"v4l2 loopback"`, `"uvcvideo"`).
54pub fn driver_name(path: &str) -> Option<String> {
55 let f = OpenOptions::new().read(true).write(true).open(path).ok()?;
56 let caps: ioctl::Capability = ioctl::querycap(&f).ok()?;
57 Some(caps.driver.clone())
58}
59
60/// True when the device is a v4l2loopback node (OBS Virtual Camera etc.).
61/// Those reject v4l2r's dqbuf (memory=0) with EINVAL — route them to
62/// [`V4l2MmapCapture`] instead of rusty-capture.
63pub fn is_loopback_device(path: &str) -> bool {
64 driver_name(path)
65 .map(|d| d.to_ascii_lowercase().contains("loopback"))
66 .unwrap_or(false)
67}
68
69/// V4L2 MMAP capture with a spec-correct dqbuf. Produces RGBA frames.
70pub struct V4l2MmapCapture {
71 device_path: String,
72 name: String,
73 width: u32,
74 height: u32,
75 fourcc: [u8; 4],
76 state: Option<CaptureState>,
77}
78
79struct CaptureState {
80 dev: File,
81 bufs: Vec<MappedBuf>,
82 started: Instant,
83}
84
85impl V4l2MmapCapture {
86 /// Open `path` and negotiate `w`×`h` (driver may adjust, as v4l2loopback
87 /// does to match the OBS output). Supports YUYV and MJPG payloads.
88 pub fn open(path: &str, w: u32, h: u32) -> Result<Self> {
89 let mut dev = OpenOptions::new()
90 .read(true)
91 .write(true)
92 .open(path)
93 .with_context(|| format!("open {path}"))?;
94 let caps: ioctl::Capability = ioctl::querycap(&dev).context("querycap")?;
95
96 // Try requested size with YUYV first (loopback native), then MJPG,
97 // then let the driver pick (0x0 keeps current).
98 let mut actual: Option<Format> = None;
99 for (req_w, req_h, fourcc) in [
100 (w, h, *b"YUYV"),
101 (w, h, *b"MJPG"),
102 (0, 0, *b"YUYV"),
103 ] {
104 let desired = Format {
105 width: req_w,
106 height: req_h,
107 pixelformat: V4l2PixelFormat::from_fourcc(&fourcc),
108 plane_fmt: vec![],
109 };
110 match ioctl::s_fmt(&mut dev, (QueueType::VideoCapture, &desired)) {
111 Ok(f) => {
112 actual = Some(f);
113 break;
114 }
115 Err(e) => log::debug!("v4l2cam: s_fmt {fourcc:?} {req_w}x{req_h}: {e}"),
116 }
117 }
118 let actual = actual.context("no acceptable V4L2 format")?;
119 let fourcc = actual.pixelformat.to_fourcc();
120 anyhow::ensure!(
121 fourcc == *b"YUYV" || fourcc == *b"MJPG",
122 "unsupported pixel format {:?} (need YUYV or MJPG)",
123 fourcc
124 );
125 log::info!(
126 "v4l2cam: opened {path} ({} / {}) {}x{} {:?}",
127 caps.card,
128 caps.driver,
129 actual.width,
130 actual.height,
131 fourcc
132 );
133 Ok(Self {
134 device_path: path.to_string(),
135 name: caps.card.clone(),
136 width: actual.width,
137 height: actual.height,
138 fourcc,
139 state: None,
140 })
141 }
142
143 fn start_streaming(&mut self) -> Result<()> {
144 if self.state.is_some() {
145 return Ok(());
146 }
147 let mut dev = OpenOptions::new()
148 .read(true)
149 .write(true)
150 .open(&self.device_path)
151 .context("reopen for streaming")?;
152 // Re-assert the negotiated format on the fresh fd.
153 let desired = Format {
154 width: self.width,
155 height: self.height,
156 pixelformat: V4l2PixelFormat::from_fourcc(&self.fourcc),
157 plane_fmt: vec![],
158 };
159 let _: Format = ioctl::s_fmt(&mut dev, (QueueType::VideoCapture, &desired))?;
160
161 let num_bufs = ioctl::reqbufs(
162 &dev,
163 QueueType::VideoCapture,
164 MemoryType::Mmap,
165 4,
166 ioctl::MemoryConsistency::empty(),
167 )
168 .context("reqbufs")?;
169 anyhow::ensure!(num_bufs > 0, "reqbufs returned 0 buffers");
170
171 let mut bufs = Vec::with_capacity(num_bufs);
172 for i in 0..num_bufs {
173 let info: QueryBuffer = ioctl::querybuf(&dev, QueueType::VideoCapture, i)?;
174 let plane = info.planes.first().context("no plane")?;
175 // v4l2r's PlaneMapping is !Send; use libc mmap so the capturer can
176 // move to the encoder thread (VideoSource: Send).
177 let ptr = unsafe {
178 libc::mmap(
179 std::ptr::null_mut(),
180 plane.length as usize,
181 libc::PROT_READ | libc::PROT_WRITE,
182 libc::MAP_SHARED,
183 dev.as_raw_fd(),
184 plane.mem_offset as libc::off_t,
185 )
186 };
187 anyhow::ensure!(ptr != libc::MAP_FAILED, "mmap buffer {i} failed");
188 bufs.push(MappedBuf {
189 ptr: ptr.cast(),
190 len: plane.length as usize,
191 });
192
193 let mut qbuf = ioctl::QBuffer::<MmapHandle>::new(QueueType::VideoCapture, i as u32);
194 qbuf.planes.push(ioctl::QBufPlane::new(0));
195 ioctl::qbuf::<_, ()>(&dev, qbuf).context("qbuf")?;
196 }
197
198 ioctl::streamon(&dev, QueueType::VideoCapture).context("streamon")?;
199 self.state = Some(CaptureState {
200 dev,
201 bufs,
202 started: Instant::now(),
203 });
204 log::debug!("v4l2cam: streaming started on {}", self.device_path);
205 Ok(())
206 }
207
208 fn stop_streaming(&mut self) {
209 if let Some(state) = self.state.take() {
210 ioctl::streamoff(&state.dev, QueueType::VideoCapture).ok();
211 log::debug!("v4l2cam: streaming stopped on {}", self.device_path);
212 }
213 }
214
215 /// dqbuf with `memory = V4L2_MEMORY_MMAP` (the field v4l2r leaves at 0,
216 /// which v4l2loopback rejects with EINVAL). Returns buffer index + bytes.
217 fn dqbuf_mmap(dev: &File) -> Result<Option<(usize, usize)>> {
218 let mut raw: bindings::v4l2_buffer = unsafe { std::mem::zeroed() };
219 raw.type_ = QueueType::VideoCapture as u32;
220 raw.memory = V4L2_MEMORY_MMAP_U32;
221 let ret = unsafe { libc::ioctl(dev.as_raw_fd(), VIDIOC_DQBUF as _, &mut raw as *mut _) };
222 if ret == 0 {
223 return Ok(Some((raw.index as usize, raw.bytesused as usize)));
224 }
225 let e = std::io::Error::last_os_error();
226 match e.raw_os_error() {
227 Some(libc::EAGAIN) => Ok(None),
228 _ => Err(anyhow::anyhow!("dqbuf: {e}")),
229 }
230 }
231
232 fn qbuf_mmap(dev: &File, index: usize) -> Result<()> {
233 let mut qbuf = ioctl::QBuffer::<MmapHandle>::new(QueueType::VideoCapture, index as u32);
234 qbuf.planes.push(ioctl::QBufPlane::new(0));
235 ioctl::qbuf::<_, ()>(dev, qbuf).context("re-queue buffer")?;
236 Ok(())
237 }
238}
239
240impl VideoSource for V4l2MmapCapture {
241 fn name(&self) -> &str {
242 &self.name
243 }
244
245 fn format(&self) -> VideoFormat {
246 VideoFormat {
247 pixel_format: PixelFormat::Rgba,
248 dimensions: [self.width, self.height],
249 }
250 }
251
252 fn start(&mut self) -> Result<()> {
253 self.start_streaming()
254 }
255
256 fn stop(&mut self) -> Result<()> {
257 self.stop_streaming();
258 Ok(())
259 }
260
261 fn pop_frame(&mut self) -> Result<Option<VideoFrame>> {
262 let Some(state) = &self.state else {
263 return Ok(None);
264 };
265 let Some((idx, bytesused)) = Self::dqbuf_mmap(&state.dev)? else {
266 return Ok(None);
267 };
268 let ts = state.started.elapsed();
269 let buf = &state.bufs[idx];
270 let n = bytesused.min(buf.len);
271 // SAFETY: buffer is dequeued to us until we re-queue below.
272 let data: &[u8] = unsafe { std::slice::from_raw_parts(buf.ptr, n) };
273
274 let rgba = if self.fourcc == *b"YUYV" {
275 Some(yuyv_to_rgba(data, self.width, self.height))
276 } else if self.fourcc == *b"MJPG" {
277 match image::load_from_memory_with_format(data, image::ImageFormat::Jpeg) {
278 Ok(img) => {
279 let rgba = img.to_rgba8();
280 if rgba.width() == self.width && rgba.height() == self.height {
281 Some(rgba.into_raw())
282 } else {
283 log::warn!("v4l2cam: MJPG dims mismatch, dropping frame");
284 None
285 }
286 }
287 Err(e) => {
288 log::warn!("v4l2cam: MJPG decode failed: {e}");
289 None
290 }
291 }
292 } else {
293 None
294 };
295
296 Self::qbuf_mmap(&state.dev, idx)?;
297
298 let Some(rgba) = rgba else {
299 return Ok(None);
300 };
301 Ok(Some(VideoFrame::new_rgba(
302 rgba.into(),
303 self.width,
304 self.height,
305 ts,
306 )))
307 }
308}
309
310impl Drop for V4l2MmapCapture {
311 fn drop(&mut self) {
312 self.stop_streaming();
313 }
314}
315
316/// Packed YUYV 4:2:2 → RGBA8 (BT.601 limited range, same as rusty-capture).
317fn yuyv_to_rgba(data: &[u8], width: u32, height: u32) -> Vec<u8> {
318 let npix = (width as usize) * (height as usize);
319 let mut rgba = vec![255u8; npix * 4];
320 let pairs = npix / 2;
321 for p in 0..pairs {
322 let o = p * 4;
323 if o + 3 >= data.len() {
324 break;
325 }
326 let y0 = data[o] as i32;
327 let u = data[o + 1] as i32 - 128;
328 let y1 = data[o + 2] as i32;
329 let v = data[o + 3] as i32 - 128;
330 let (r0, g0, b0) = yuv_to_rgb(y0, u, v);
331 let (r1, g1, b1) = yuv_to_rgb(y1, u, v);
332 let px = p * 8;
333 rgba[px] = r0;
334 rgba[px + 1] = g0;
335 rgba[px + 2] = b0;
336 rgba[px + 4] = r1;
337 rgba[px + 5] = g1;
338 rgba[px + 6] = b1;
339 }
340 rgba
341}
342
343fn yuv_to_rgb(y: i32, u: i32, v: i32) -> (u8, u8, u8) {
344 // BT.601 studio swing: C = 1.164(Y-16)
345 let c = (298 * (y - 16) + 128) >> 8;
346 let r = (c + ((409 * v + 128) >> 8)).clamp(0, 255) as u8;
347 let g = (c - ((100 * u + 208 * v + 128) >> 8)).clamp(0, 255) as u8;
348 let b = (c + ((516 * u + 128) >> 8)).clamp(0, 255) as u8;
349 (r, g, b)
350}
351
352#[cfg(test)]
353mod tests {
354 use super::*;
355 use std::time::Duration;
356
357 #[test]
358 fn yuyv_mid_gray_is_achromatic() {
359 // Y=128 U=128 V=128 → mid gray, R=G=B≈128, alpha forced 255.
360 let data = [128u8, 128, 128, 128];
361 let rgba = yuyv_to_rgba(&data, 2, 1);
362 assert_eq!(rgba.len(), 2 * 4);
363 let (r, g, b, a) = (rgba[0], rgba[1], rgba[2], rgba[3]);
364 assert_eq!(a, 255);
365 assert!((r as i16 - g as i16).abs() <= 2, "r={r} g={g}");
366 assert!((g as i16 - b as i16).abs() <= 2, "g={g} b={b}");
367 assert!((r as i16 - 128).abs() <= 4, "expected ~128, got {r}");
368 }
369
370 #[test]
371 fn yuyv_color_pair_preserves_difference() {
372 // Two luma levels must produce different RGB (real content, not uniform).
373 let data = [60u8, 90, 200, 90];
374 let rgba = yuyv_to_rgba(&data, 2, 1);
375 assert!(rgba[0] != rgba[4], "different Y must give different R");
376 }
377
378 /// Real-device integration proof: capture one frame from the loopback
379 /// device (or SLEEK_TEST_CAMERA_ID) via THIS shipped capture path and
380 /// assert non-uniform content. Skips with a reason when unavailable.
381 #[test]
382 fn v4l2_capture_yields_nonuniform_frame() {
383 let id = std::env::var("SLEEK_TEST_CAMERA_ID")
384 .ok()
385 .filter(|s| !s.is_empty())
386 .unwrap_or_else(|| "/dev/video10".to_string());
387 let mut cam = match V4l2MmapCapture::open(&id, 640, 360) {
388 Ok(c) => c,
389 Err(e) => {
390 eprintln!("SKIP v4l2_capture id={id}: open failed: {e:#}");
391 return;
392 }
393 };
394 if let Err(e) = cam.start() {
395 eprintln!("SKIP v4l2_capture id={id}: start failed: {e:#}");
396 return;
397 }
398 let deadline = Instant::now() + Duration::from_millis(2_500);
399 let mut frame = None;
400 while Instant::now() < deadline {
401 match cam.pop_frame() {
402 Ok(Some(f)) => {
403 frame = Some(f);
404 break;
405 }
406 Ok(None) => std::thread::sleep(Duration::from_millis(20)),
407 Err(e) => {
408 let _ = cam.stop();
409 eprintln!("SKIP v4l2_capture id={id}: pop_frame: {e:#}");
410 return;
411 }
412 }
413 }
414 let _ = cam.stop();
415 let Some(f) = frame else {
416 eprintln!("SKIP v4l2_capture id={id}: no frames within 2.5s");
417 return;
418 };
419 let rgba = f.rgba_image();
420 let bytes = rgba.as_raw().as_slice();
421 let mut min = 255u8;
422 let mut max = 0u8;
423 for px in bytes.chunks_exact(4) {
424 let l = ((77u32 * px[0] as u32 + 150u32 * px[1] as u32 + 29u32 * px[2] as u32) >> 8) as u8;
425 min = min.min(l);
426 max = max.max(l);
427 }
428 eprintln!(
429 "v4l2_capture id={id} {}x{} luma {min}..{max} (alpha0={})",
430 f.width(),
431 f.height(),
432 bytes.get(3).copied().unwrap_or(0)
433 );
434 assert!(
435 max > min,
436 "captured frame must be non-uniform (luma {min}..{max})"
437 );
438 }
439}