rickub
nandi/jolt-nativepublic⑂ Fork 0
⑂ 789bb134c91bfdee346df2dc9656bf2e7f9bbd1a
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/jolt-native.git
git clone ssh://git@rickub.com/nandi/jolt-native.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

v4l2cam.rs · 450 lines · 15.4 KBRust Blame HistoryRaw
Clear the clippy backlog the new CI enforces 4956d1e nandi 13d ago1// This file tracks sleek's copy in `android/src` closely enough that a fix can
2// be moved between the two by eye, so it is deliberately not idiomatised to
3// this workspace's clippy settings. The lints below are the ones that would
4// rewrite it away from its original; everything else still applies.
5#![allow(
6 clippy::chunks_exact_to_as_chunks,
7 clippy::identity_op,
8 clippy::manual_filter,
9 clippy::manual_is_multiple_of,
10 clippy::redundant_closure,
11 clippy::too_many_arguments,
12 clippy::unnecessary_sort_by
13)]
14
Lift freeq's AV media plane out of sleek 90f8b89 nandi 19d ago15//! Minimal V4L2 MMAP camera capture with a **correct `dqbuf`**.
16//!
17//! `v4l2r`'s `ioctl::dqbuf` leaves `v4l2_buffer.memory = 0`. Real UVC drivers
18//! (EMEET) tolerate it, but **v4l2loopback (OBS Virtual Camera) rejects it with
19//! `EINVAL`**, which kills the capture thread on the first frame and leaves a
20//! permanently blank self-view tile. This module does the same MMAP flow with
21//! `memory = V4L2_MEMORY_MMAP` set explicitly — proven against `/dev/video10`
22//! (OBS) with `examples/v4l2_probe.rs`.
23//!
24//! Only used for loopback/virtual devices; hardware cams stay on
25//! `rusty-capture`'s richer capturer (more pixel formats, zero-copy paths).
26
27use std::fs::{File, OpenOptions};
28use std::os::unix::io::AsRawFd;
29use std::time::Instant;
30
31use anyhow::{Context, Result};
32use iroh_live::media::format::{PixelFormat, VideoFormat, VideoFrame};
33use iroh_live::media::traits::VideoSource;
34use v4l2r::bindings;
35use v4l2r::ioctl::{self, QueryBuffer};
36use v4l2r::memory::{MemoryType, MmapHandle};
37use v4l2r::{Format, PixelFormat as V4l2PixelFormat, QueueType};
38
39const V4L2_MEMORY_MMAP_U32: u32 = 1;
40/// VIDIOC_DQBUF = _IOWR('V', 17, struct v4l2_buffer). v4l2r's dqbuf helper is
41/// unusable for loopback devices (memory field never set), so issue it directly.
42const VIDIOC_DQBUF: std::ffi::c_ulong = (3 << 30)
43 | (('V' as std::ffi::c_ulong) << 8)
44 | 17
45 | ((std::mem::size_of::<bindings::v4l2_buffer>() as std::ffi::c_ulong) << 16);
46
47/// One mmap'd capture buffer (kernel-owned, re-queued after each frame).
48struct MappedBuf {
49 ptr: *mut u8,
50 len: usize,
51}
52
53// SAFETY: the mapping is process-shared memory from the kernel; access is
54// synchronized by dqbuf/qbuf ownership (we only read while dequeued to us).
55unsafe impl Send for MappedBuf {}
56
57impl Drop for MappedBuf {
58 fn drop(&mut self) {
59 if !self.ptr.is_null() && self.len > 0 {
60 unsafe {
61 libc::munmap(self.ptr.cast(), self.len);
62 }
63 }
64 }
65}
66
67/// Read the V4L2 driver name (e.g. `"v4l2 loopback"`, `"uvcvideo"`).
68pub fn driver_name(path: &str) -> Option<String> {
69 let f = OpenOptions::new().read(true).write(true).open(path).ok()?;
70 let caps: ioctl::Capability = ioctl::querycap(&f).ok()?;
71 Some(caps.driver.clone())
72}
73
74/// True when the device is a v4l2loopback node (OBS Virtual Camera etc.).
75/// Those reject v4l2r's dqbuf (memory=0) with EINVAL — route them to
76/// [`V4l2MmapCapture`] instead of rusty-capture.
77pub fn is_loopback_device(path: &str) -> bool {
78 driver_name(path)
79 .map(|d| d.to_ascii_lowercase().contains("loopback"))
80 .unwrap_or(false)
81}
82
83/// V4L2 MMAP capture with a spec-correct dqbuf. Produces RGBA frames.
84pub struct V4l2MmapCapture {
85 device_path: String,
86 name: String,
87 width: u32,
88 height: u32,
89 fourcc: [u8; 4],
90 state: Option<CaptureState>,
91}
92
93struct CaptureState {
94 dev: File,
95 bufs: Vec<MappedBuf>,
96 started: Instant,
97}
98
99impl V4l2MmapCapture {
100 /// Open `path` and negotiate `w`×`h` (driver may adjust, as v4l2loopback
101 /// does to match the OBS output). Supports YUYV and MJPG payloads.
102 pub fn open(path: &str, w: u32, h: u32) -> Result<Self> {
103 let mut dev = OpenOptions::new()
104 .read(true)
105 .write(true)
106 .open(path)
107 .with_context(|| format!("open {path}"))?;
108 let caps: ioctl::Capability = ioctl::querycap(&dev).context("querycap")?;
109
110 // Try requested size with YUYV first (loopback native), then MJPG,
111 // then let the driver pick (0x0 keeps current).
112 let mut actual: Option<Format> = None;
Run the formatter over the tree 3e8c6f0 nandi 13d ago113 for (req_w, req_h, fourcc) in [(w, h, *b"YUYV"), (w, h, *b"MJPG"), (0, 0, *b"YUYV")] {
Lift freeq's AV media plane out of sleek 90f8b89 nandi 19d ago114 let desired = Format {
115 width: req_w,
116 height: req_h,
117 pixelformat: V4l2PixelFormat::from_fourcc(&fourcc),
118 plane_fmt: vec![],
119 };
120 match ioctl::s_fmt(&mut dev, (QueueType::VideoCapture, &desired)) {
121 Ok(f) => {
122 actual = Some(f);
123 break;
124 }
125 Err(e) => log::debug!("v4l2cam: s_fmt {fourcc:?} {req_w}x{req_h}: {e}"),
126 }
127 }
128 let actual = actual.context("no acceptable V4L2 format")?;
129 let fourcc = actual.pixelformat.to_fourcc();
130 anyhow::ensure!(
131 fourcc == *b"YUYV" || fourcc == *b"MJPG",
132 "unsupported pixel format {:?} (need YUYV or MJPG)",
133 fourcc
134 );
135 log::info!(
136 "v4l2cam: opened {path} ({} / {}) {}x{} {:?}",
137 caps.card,
138 caps.driver,
139 actual.width,
140 actual.height,
141 fourcc
142 );
143 Ok(Self {
144 device_path: path.to_string(),
145 name: caps.card.clone(),
146 width: actual.width,
147 height: actual.height,
148 fourcc,
149 state: None,
150 })
151 }
152
153 fn start_streaming(&mut self) -> Result<()> {
154 if self.state.is_some() {
155 return Ok(());
156 }
157 let mut dev = OpenOptions::new()
158 .read(true)
159 .write(true)
160 .open(&self.device_path)
161 .context("reopen for streaming")?;
162 // Re-assert the negotiated format on the fresh fd.
163 let desired = Format {
164 width: self.width,
165 height: self.height,
166 pixelformat: V4l2PixelFormat::from_fourcc(&self.fourcc),
167 plane_fmt: vec![],
168 };
169 let _: Format = ioctl::s_fmt(&mut dev, (QueueType::VideoCapture, &desired))?;
170
171 let num_bufs = ioctl::reqbufs(
172 &dev,
173 QueueType::VideoCapture,
174 MemoryType::Mmap,
175 4,
176 ioctl::MemoryConsistency::empty(),
177 )
178 .context("reqbufs")?;
179 anyhow::ensure!(num_bufs > 0, "reqbufs returned 0 buffers");
180
181 let mut bufs = Vec::with_capacity(num_bufs);
182 for i in 0..num_bufs {
183 let info: QueryBuffer = ioctl::querybuf(&dev, QueueType::VideoCapture, i)?;
184 let plane = info.planes.first().context("no plane")?;
185 // v4l2r's PlaneMapping is !Send; use libc mmap so the capturer can
186 // move to the encoder thread (VideoSource: Send).
187 let ptr = unsafe {
188 libc::mmap(
189 std::ptr::null_mut(),
190 plane.length as usize,
191 libc::PROT_READ | libc::PROT_WRITE,
192 libc::MAP_SHARED,
193 dev.as_raw_fd(),
194 plane.mem_offset as libc::off_t,
195 )
196 };
197 anyhow::ensure!(ptr != libc::MAP_FAILED, "mmap buffer {i} failed");
198 bufs.push(MappedBuf {
199 ptr: ptr.cast(),
200 len: plane.length as usize,
201 });
202
203 let mut qbuf = ioctl::QBuffer::<MmapHandle>::new(QueueType::VideoCapture, i as u32);
204 qbuf.planes.push(ioctl::QBufPlane::new(0));
205 ioctl::qbuf::<_, ()>(&dev, qbuf).context("qbuf")?;
206 }
207
208 ioctl::streamon(&dev, QueueType::VideoCapture).context("streamon")?;
209 self.state = Some(CaptureState {
210 dev,
211 bufs,
212 started: Instant::now(),
213 });
214 log::debug!("v4l2cam: streaming started on {}", self.device_path);
215 Ok(())
216 }
217
218 fn stop_streaming(&mut self) {
219 if let Some(state) = self.state.take() {
220 ioctl::streamoff(&state.dev, QueueType::VideoCapture).ok();
221 log::debug!("v4l2cam: streaming stopped on {}", self.device_path);
222 }
223 }
224
225 /// dqbuf with `memory = V4L2_MEMORY_MMAP` (the field v4l2r leaves at 0,
226 /// which v4l2loopback rejects with EINVAL). Returns buffer index + bytes.
227 fn dqbuf_mmap(dev: &File) -> Result<Option<(usize, usize)>> {
228 let mut raw: bindings::v4l2_buffer = unsafe { std::mem::zeroed() };
229 raw.type_ = QueueType::VideoCapture as u32;
230 raw.memory = V4L2_MEMORY_MMAP_U32;
231 let ret = unsafe { libc::ioctl(dev.as_raw_fd(), VIDIOC_DQBUF as _, &mut raw as *mut _) };
232 if ret == 0 {
233 return Ok(Some((raw.index as usize, raw.bytesused as usize)));
234 }
235 let e = std::io::Error::last_os_error();
236 match e.raw_os_error() {
237 Some(libc::EAGAIN) => Ok(None),
238 _ => Err(anyhow::anyhow!("dqbuf: {e}")),
239 }
240 }
241
242 fn qbuf_mmap(dev: &File, index: usize) -> Result<()> {
243 let mut qbuf = ioctl::QBuffer::<MmapHandle>::new(QueueType::VideoCapture, index as u32);
244 qbuf.planes.push(ioctl::QBufPlane::new(0));
245 ioctl::qbuf::<_, ()>(dev, qbuf).context("re-queue buffer")?;
246 Ok(())
247 }
248}
249
250impl VideoSource for V4l2MmapCapture {
251 fn name(&self) -> &str {
252 &self.name
253 }
254
255 fn format(&self) -> VideoFormat {
256 VideoFormat {
257 pixel_format: PixelFormat::Rgba,
258 dimensions: [self.width, self.height],
259 }
260 }
261
262 fn start(&mut self) -> Result<()> {
263 self.start_streaming()
264 }
265
266 fn stop(&mut self) -> Result<()> {
267 self.stop_streaming();
268 Ok(())
269 }
270
271 fn pop_frame(&mut self) -> Result<Option<VideoFrame>> {
272 let Some(state) = &self.state else {
273 return Ok(None);
274 };
275 let Some((idx, bytesused)) = Self::dqbuf_mmap(&state.dev)? else {
276 return Ok(None);
277 };
278 let ts = state.started.elapsed();
279 let buf = &state.bufs[idx];
280 let n = bytesused.min(buf.len);
281 // SAFETY: buffer is dequeued to us until we re-queue below.
282 let data: &[u8] = unsafe { std::slice::from_raw_parts(buf.ptr, n) };
283
284 let rgba = if self.fourcc == *b"YUYV" {
285 Some(yuyv_to_rgba(data, self.width, self.height))
286 } else if self.fourcc == *b"MJPG" {
287 match image::load_from_memory_with_format(data, image::ImageFormat::Jpeg) {
288 Ok(img) => {
289 let rgba = img.to_rgba8();
290 if rgba.width() == self.width && rgba.height() == self.height {
291 Some(rgba.into_raw())
292 } else {
293 log::warn!("v4l2cam: MJPG dims mismatch, dropping frame");
294 None
295 }
296 }
297 Err(e) => {
298 log::warn!("v4l2cam: MJPG decode failed: {e}");
299 None
300 }
301 }
302 } else {
303 None
304 };
305
306 Self::qbuf_mmap(&state.dev, idx)?;
307
308 let Some(rgba) = rgba else {
309 return Ok(None);
310 };
311 Ok(Some(VideoFrame::new_rgba(
312 rgba.into(),
313 self.width,
314 self.height,
315 ts,
316 )))
317 }
318}
319
320impl Drop for V4l2MmapCapture {
321 fn drop(&mut self) {
322 self.stop_streaming();
323 }
324}
325
326/// Packed YUYV 4:2:2 → RGBA8 (BT.601 limited range, same as rusty-capture).
327fn yuyv_to_rgba(data: &[u8], width: u32, height: u32) -> Vec<u8> {
328 let npix = (width as usize) * (height as usize);
329 let mut rgba = vec![255u8; npix * 4];
330 let pairs = npix / 2;
331 for p in 0..pairs {
332 let o = p * 4;
333 if o + 3 >= data.len() {
334 break;
335 }
336 let y0 = data[o] as i32;
337 let u = data[o + 1] as i32 - 128;
338 let y1 = data[o + 2] as i32;
339 let v = data[o + 3] as i32 - 128;
340 let (r0, g0, b0) = yuv_to_rgb(y0, u, v);
341 let (r1, g1, b1) = yuv_to_rgb(y1, u, v);
342 let px = p * 8;
343 rgba[px] = r0;
344 rgba[px + 1] = g0;
345 rgba[px + 2] = b0;
346 rgba[px + 4] = r1;
347 rgba[px + 5] = g1;
348 rgba[px + 6] = b1;
349 }
350 rgba
351}
352
353fn yuv_to_rgb(y: i32, u: i32, v: i32) -> (u8, u8, u8) {
354 // BT.601 studio swing: C = 1.164(Y-16)
355 let c = (298 * (y - 16) + 128) >> 8;
356 let r = (c + ((409 * v + 128) >> 8)).clamp(0, 255) as u8;
357 let g = (c - ((100 * u + 208 * v + 128) >> 8)).clamp(0, 255) as u8;
358 let b = (c + ((516 * u + 128) >> 8)).clamp(0, 255) as u8;
359 (r, g, b)
360}
361
362#[cfg(test)]
363mod tests {
364 use super::*;
365 use std::time::Duration;
366
367 #[test]
368 fn yuyv_mid_gray_is_achromatic() {
369 // Y=128 U=128 V=128 → mid gray, R=G=B≈128, alpha forced 255.
370 let data = [128u8, 128, 128, 128];
371 let rgba = yuyv_to_rgba(&data, 2, 1);
372 assert_eq!(rgba.len(), 2 * 4);
373 let (r, g, b, a) = (rgba[0], rgba[1], rgba[2], rgba[3]);
374 assert_eq!(a, 255);
375 assert!((r as i16 - g as i16).abs() <= 2, "r={r} g={g}");
376 assert!((g as i16 - b as i16).abs() <= 2, "g={g} b={b}");
377 assert!((r as i16 - 128).abs() <= 4, "expected ~128, got {r}");
378 }
379
380 #[test]
381 fn yuyv_color_pair_preserves_difference() {
382 // Two luma levels must produce different RGB (real content, not uniform).
383 let data = [60u8, 90, 200, 90];
384 let rgba = yuyv_to_rgba(&data, 2, 1);
385 assert!(rgba[0] != rgba[4], "different Y must give different R");
386 }
387
388 /// Real-device integration proof: capture one frame from the loopback
389 /// device (or SLEEK_TEST_CAMERA_ID) via THIS shipped capture path and
390 /// assert non-uniform content. Skips with a reason when unavailable.
391 #[test]
392 fn v4l2_capture_yields_nonuniform_frame() {
393 let id = std::env::var("SLEEK_TEST_CAMERA_ID")
394 .ok()
395 .filter(|s| !s.is_empty())
396 .unwrap_or_else(|| "/dev/video10".to_string());
397 let mut cam = match V4l2MmapCapture::open(&id, 640, 360) {
398 Ok(c) => c,
399 Err(e) => {
400 eprintln!("SKIP v4l2_capture id={id}: open failed: {e:#}");
401 return;
402 }
403 };
404 if let Err(e) = cam.start() {
405 eprintln!("SKIP v4l2_capture id={id}: start failed: {e:#}");
406 return;
407 }
408 let deadline = Instant::now() + Duration::from_millis(2_500);
409 let mut frame = None;
410 while Instant::now() < deadline {
411 match cam.pop_frame() {
412 Ok(Some(f)) => {
413 frame = Some(f);
414 break;
415 }
416 Ok(None) => std::thread::sleep(Duration::from_millis(20)),
417 Err(e) => {
418 let _ = cam.stop();
419 eprintln!("SKIP v4l2_capture id={id}: pop_frame: {e:#}");
420 return;
421 }
422 }
423 }
424 let _ = cam.stop();
425 let Some(f) = frame else {
426 eprintln!("SKIP v4l2_capture id={id}: no frames within 2.5s");
427 return;
428 };
429 let rgba = f.rgba_image();
430 let bytes = rgba.as_raw().as_slice();
431 let mut min = 255u8;
432 let mut max = 0u8;
433 for px in bytes.chunks_exact(4) {
Run the formatter over the tree 3e8c6f0 nandi 13d ago434 let l =
435 ((77u32 * px[0] as u32 + 150u32 * px[1] as u32 + 29u32 * px[2] as u32) >> 8) as u8;
Lift freeq's AV media plane out of sleek 90f8b89 nandi 19d ago436 min = min.min(l);
437 max = max.max(l);
438 }
439 eprintln!(
440 "v4l2_capture id={id} {}x{} luma {min}..{max} (alpha0={})",
441 f.width(),
442 f.height(),
443 bytes.get(3).copied().unwrap_or(0)
444 );
445 assert!(
446 max > min,
447 "captured frame must be non-uniform (luma {min}..{max})"
448 );
449 }
450}