rickub
nandi/jolt-nativepublic⑂ Fork 0
⑂ 789bb134c91bfdee346df2dc9656bf2e7f9bbd1a
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/jolt-native.git
git clone ssh://git@rickub.com/nandi/jolt-native.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

android_camera.rs · 428 lines · 13.9 KBRust Blame HistoryRaw
Clear the clippy backlog the new CI enforces 4956d1e nandi 13d ago1// This file tracks sleek's copy in `android/src` closely enough that a fix can
2// be moved between the two by eye, so it is deliberately not idiomatised to
3// this workspace's clippy settings. The lints below are the ones that would
4// rewrite it away from its original; everything else still applies.
5#![allow(
6 clippy::chunks_exact_to_as_chunks,
7 clippy::identity_op,
8 clippy::manual_filter,
9 clippy::manual_is_multiple_of,
10 clippy::redundant_closure,
11 clippy::too_many_arguments,
12 clippy::unnecessary_sort_by
13)]
14
Let the media plane cross to the phone, camera and all fd0e21a nandi 18d ago15//! Android Camera2 → MoQ video publish bridge.
16//!
17//! Java `CameraCapture` (in APK `classes.dex`) opens Camera2 / ImageReader and
18//! calls native NV12 push methods implemented here. Frames land in a
19//! latest-only [`PushCameraSource`] that implements iroh-live's [`VideoSource`].
20//!
21//! JNI note: never resolve `CameraCapture` with `Env::find_class` from a
22//! native worker thread — that uses the system ClassLoader and misses APK
23//! classes. Use [`android_jni::load_app_class`] (Activity ClassLoader).
24//!
25//! The `JavaVM` and the Activity come from [`crate::android_jni`], which the
26//! glue fills in before a call can start — see the note there for why they
27//! arrive by hand rather than out of `ndk_context`.
28
29use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
30use std::sync::{Arc, Mutex, OnceLock};
31use std::time::Duration;
32
33use anyhow::{anyhow, Context, Result};
34use iroh_live::media::{
35 format::{Nv12Planes, PixelFormat, VideoFormat, VideoFrame},
36 traits::VideoSource,
37};
38
39use crate::android_jni;
40
41/// Java binary name for the Camera2 helper in APK `classes.dex`.
42const CAMERA_CAPTURE_CLASS: &str = "uk.nandi.frq.CameraCapture";
43
44/// What every camera call says when the glue never handed the handles over —
45/// an APK packaging `libjoltmoq.so` but not calling `joltmoq_android_init`.
46const NOT_INITIALISED: &str = "joltmoq_android_init has not run";
47
48/// Shared sink written by JNI and read by the encoder thread.
49struct FrameSink {
50 pending: Mutex<Option<VideoFrame>>,
51 format: Mutex<VideoFormat>,
52 /// Set true after Java reports the capture session is live.
53 opened: AtomicBool,
54 /// Last open error detail (empty when ok / idle).
55 last_error: Mutex<String>,
56 frames_pushed: AtomicU64,
57}
58
59impl FrameSink {
60 fn new(width: u32, height: u32) -> Self {
61 Self {
62 pending: Mutex::new(None),
63 format: Mutex::new(VideoFormat {
64 // Encoders that receive FrameData::Nv12 ignore this field;
65 // keep Rgba as the VideoFormat default (rusty-codecs has no Nv12 variant).
66 pixel_format: PixelFormat::Rgba,
67 dimensions: [width, height],
68 }),
69 opened: AtomicBool::new(false),
70 last_error: Mutex::new(String::new()),
71 frames_pushed: AtomicU64::new(0),
72 }
73 }
74}
75
76static SINK: OnceLock<Arc<FrameSink>> = OnceLock::new();
77
78fn sink() -> Arc<FrameSink> {
79 SINK.get_or_init(|| Arc::new(FrameSink::new(640, 480)))
80 .clone()
81}
82
83/// Latest-frame-only [`VideoSource`] fed by Camera2 JNI callbacks.
84pub struct PushCameraSource {
85 sink: Arc<FrameSink>,
86 name: String,
87}
88
89impl PushCameraSource {
90 pub fn new(label: impl Into<String>) -> Self {
91 Self {
92 sink: sink(),
93 name: label.into(),
94 }
95 }
96}
97
98impl VideoSource for PushCameraSource {
99 fn name(&self) -> &str {
100 &self.name
101 }
102
103 fn format(&self) -> VideoFormat {
104 self.sink
105 .format
106 .lock()
107 .map(|g| g.clone())
108 .unwrap_or(VideoFormat {
109 pixel_format: PixelFormat::Rgba,
110 dimensions: [640, 480],
111 })
112 }
113
114 fn pop_frame(&mut self) -> Result<Option<VideoFrame>> {
115 Ok(self.sink.pending.lock().ok().and_then(|mut g| g.take()))
116 }
117
118 fn start(&mut self) -> Result<()> {
119 Ok(())
120 }
121
122 fn stop(&mut self) -> Result<()> {
123 if let Ok(mut g) = self.sink.pending.lock() {
124 *g = None;
125 }
126 Ok(())
127 }
128}
129
130/// Enumerate cameras for the device picker (front first).
131/// Returns `(id, label)` pairs.
132pub fn list_cameras() -> Vec<(String, String)> {
133 match list_cameras_jni() {
134 Ok(list) => list,
135 Err(e) => {
136 // Warn: empty list makes AV report "camera unavailable" permanently.
137 log::warn!("android camera list: {e}");
138 Vec::new()
139 }
140 }
141}
142
143fn list_cameras_jni() -> Result<Vec<(String, String)>> {
144 let vm = android_jni::vm().context(NOT_INITIALISED)?;
145 let activity_ptr = android_jni::activity().context(NOT_INITIALISED)?;
146
147 use jni::objects::{JObject, JObjectArray, JString, JValue};
148 use jni::refs::Global;
149 use jni::{jni_sig, jni_str};
150
151 let mut out: Option<Result<Vec<(String, String)>>> = None;
152 vm.attach_current_thread(|env| -> jni::errors::Result<()> {
153 let activity = unsafe { env.as_cast_raw::<Global<JObject>>(&activity_ptr)? };
154 let cls = match android_jni::load_app_class(env, activity.as_ref(), CAMERA_CAPTURE_CLASS) {
155 Ok(c) => c,
156 Err(e) => {
157 out = Some(Err(anyhow!("CameraCapture class: {e}")));
158 return Ok(());
159 }
160 };
161 let arr_obj = env
162 .call_static_method(
163 &cls,
164 jni_str!("listCameras"),
165 jni_sig!((android.content.Context) -> [java.lang.String]),
166 &[JValue::Object(activity.as_ref())],
167 )?
168 .l()?;
169 let arr = env.cast_local::<JObjectArray>(arr_obj)?;
170 let n = arr.len(env)?;
171 let mut list = Vec::with_capacity(n);
172 for i in 0..n {
173 let obj = arr.get_element(env, i)?;
174 if obj.is_null() {
175 continue;
176 }
177 let jstr = env.cast_local::<JString>(obj)?;
178 let s = format!("{jstr}");
179 let (id, name) = match s.split_once('\t') {
180 Some((id, name)) => (id.to_string(), name.to_string()),
181 None => (s.clone(), s),
182 };
183 list.push((id, name));
184 }
185 out = Some(Ok(list));
186 Ok(())
187 })
188 .map_err(|e| anyhow!("list cameras JNI: {e}"))?;
189 out.unwrap_or_else(|| Err(anyhow!("list cameras JNI: no result")))
190}
191
192/// Start Camera2 capture. Returns when the open request is dispatched (session
193/// readiness is async — wait with [`wait_until_opened`]).
194pub fn start_capture(camera_id: Option<&str>) -> Result<()> {
195 let s = sink();
196 s.opened.store(false, Ordering::Relaxed);
197 s.frames_pushed.store(0, Ordering::Relaxed);
198 if let Ok(mut e) = s.last_error.lock() {
199 e.clear();
200 }
201 if let Ok(mut p) = s.pending.lock() {
202 *p = None;
203 }
204
205 let vm = android_jni::vm().context(NOT_INITIALISED)?;
206 let activity_ptr = android_jni::activity().context(NOT_INITIALISED)?;
207
208 use jni::objects::{JObject, JValue};
209 use jni::refs::Global;
210 use jni::{jni_sig, jni_str};
211
212 let id = camera_id.unwrap_or("").to_string();
213 let mut start_err: Option<anyhow::Error> = None;
214 vm.attach_current_thread(|env| -> jni::errors::Result<()> {
215 let activity = unsafe { env.as_cast_raw::<Global<JObject>>(&activity_ptr)? };
216 let cls = match android_jni::load_app_class(env, activity.as_ref(), CAMERA_CAPTURE_CLASS) {
217 Ok(c) => c,
218 Err(e) => {
219 start_err = Some(anyhow!("CameraCapture class: {e}"));
220 return Ok(());
221 }
222 };
223 let jid = env.new_string(&id)?;
224 env.call_static_method(
225 &cls,
226 jni_str!("start"),
227 jni_sig!((android.app.Activity, java.lang.String) -> void),
228 &[JValue::Object(activity.as_ref()), JValue::Object(&jid)],
229 )?;
230 Ok(())
231 })
232 .map_err(|e| anyhow!("CameraCapture.start: {e}"))?;
233 if let Some(e) = start_err {
234 return Err(e).context("CameraCapture.start");
235 }
236 Ok(())
237}
238
239/// Block briefly until Java reports the capture session is live.
240pub fn wait_until_opened(timeout: Duration) -> Result<()> {
241 let s = sink();
242 let deadline = std::time::Instant::now() + timeout;
243 while std::time::Instant::now() < deadline {
244 if s.opened.load(Ordering::Relaxed) {
245 return Ok(());
246 }
247 let err = s
248 .last_error
249 .lock()
250 .ok()
251 .map(|g| g.clone())
252 .unwrap_or_default();
253 if !err.is_empty() {
254 return Err(anyhow!("camera open failed: {err}"));
255 }
256 std::thread::sleep(Duration::from_millis(20));
257 }
258 let err = s
259 .last_error
260 .lock()
261 .ok()
262 .map(|g| g.clone())
263 .unwrap_or_default();
264 if s.opened.load(Ordering::Relaxed) {
265 Ok(())
266 } else if !err.is_empty() {
267 Err(anyhow!("camera open failed: {err}"))
268 } else {
269 Err(anyhow!("camera open timed out"))
270 }
271}
272
273/// Stop Camera2 capture (idempotent).
274pub fn stop_capture() {
275 let (Some(vm), Some(activity_ptr)) = (android_jni::vm(), android_jni::activity()) else {
276 return;
277 };
278
279 use jni::objects::{JObject, JValue};
280 use jni::refs::Global;
281 use jni::{jni_sig, jni_str};
282
283 let _ = vm.attach_current_thread(|env| -> jni::errors::Result<()> {
284 let activity = unsafe { env.as_cast_raw::<Global<JObject>>(&activity_ptr)? };
285 let cls = match android_jni::load_app_class(env, activity.as_ref(), CAMERA_CAPTURE_CLASS) {
286 Ok(c) => c,
287 Err(e) => {
288 log::warn!("android camera stop: CameraCapture class: {e}");
289 return Ok(());
290 }
291 };
292 env.call_static_method(
293 &cls,
294 jni_str!("stop"),
295 jni_sig!((android.app.Activity) -> void),
296 &[JValue::Object(activity.as_ref())],
297 )?;
298 Ok(())
299 });
300 let s = sink();
301 s.opened.store(false, Ordering::Relaxed);
302 if let Ok(mut p) = s.pending.lock() {
303 *p = None;
304 };
305}
306
307/// RAII guard that stops Camera2 when the media session ends.
308pub struct CameraCaptureGuard;
309
310impl Drop for CameraCaptureGuard {
311 fn drop(&mut self) {
312 stop_capture();
313 log::info!("android camera: capture stopped (guard drop)");
314 }
315}
316
317// ── JNI callbacks from CameraCapture.java ─────────────────────────────────
318
319#[unsafe(no_mangle)]
320pub extern "system" fn Java_uk_nandi_frq_CameraCapture_onNv12Frame<'local>(
321 mut unowned_env: jni::JNIEnv<'local>,
322 _class: jni::objects::JClass<'local>,
323 y_data: jni::objects::JByteArray<'local>,
324 uv_data: jni::objects::JByteArray<'local>,
325 width: jni::sys::jint,
326 height: jni::sys::jint,
327 y_stride: jni::sys::jint,
328 uv_stride: jni::sys::jint,
329 rotation_degrees: jni::sys::jint,
330) {
331 if width <= 0 || height <= 0 {
332 return;
333 }
334 let _ = unowned_env
335 .with_env(|env| -> jni::errors::Result<()> {
336 let y_bytes = env.convert_byte_array(&y_data)?;
337 let uv_bytes = env.convert_byte_array(&uv_data)?;
338 let src_w = width as u32;
339 let src_h = height as u32;
340 let rot = if rotation_degrees < 0 {
341 0
342 } else {
343 rotation_degrees as u32
344 };
345 // Apply sensor/display orientation so published + preview frames
346 // are upright (Camera2 ImageReader buffers are sensor-oriented).
Run the formatter over the tree 3e8c6f0 nandi 13d ago347 let (y_bytes, uv_bytes, w, h, y_str, uv_str) = match crate::nv12_orient::orient_nv12(
348 &y_bytes,
349 &uv_bytes,
350 src_w,
351 src_h,
352 y_stride as u32,
353 uv_stride as u32,
354 rot,
355 ) {
356 Some((y, uv, w, h)) => (y, uv, w, h, w, w),
357 None => (
358 y_bytes,
359 uv_bytes,
Let the media plane cross to the phone, camera and all fd0e21a nandi 18d ago360 src_w,
361 src_h,
362 y_stride as u32,
363 uv_stride as u32,
Run the formatter over the tree 3e8c6f0 nandi 13d ago364 ),
365 };
Let the media plane cross to the phone, camera and all fd0e21a nandi 18d ago366 let frame = VideoFrame::new_nv12(
367 Nv12Planes {
368 y_data: y_bytes,
369 y_stride: y_str,
370 uv_data: uv_bytes,
371 uv_stride: uv_str,
372 width: w,
373 height: h,
374 },
375 Duration::ZERO,
376 );
377 let s = sink();
378 if let Ok(mut fmt) = s.format.lock() {
379 if fmt.dimensions != [w, h] {
380 *fmt = VideoFormat {
381 pixel_format: PixelFormat::Rgba,
382 dimensions: [w, h],
383 };
384 }
385 }
386 if let Ok(mut pending) = s.pending.lock() {
387 *pending = Some(frame);
388 }
389 let n = s.frames_pushed.fetch_add(1, Ordering::Relaxed);
390 if n == 0 {
Run the formatter over the tree 3e8c6f0 nandi 13d ago391 log::info!(
392 "android camera: first NV12 frame {w}x{h} (src {src_w}x{src_h} rot={rot})"
393 );
Let the media plane cross to the phone, camera and all fd0e21a nandi 18d ago394 }
395 Ok(())
396 })
397 .into_outcome();
398}
399
400#[unsafe(no_mangle)]
401pub extern "system" fn Java_uk_nandi_frq_CameraCapture_onCameraState<'local>(
402 mut unowned_env: jni::JNIEnv<'local>,
403 _class: jni::objects::JClass<'local>,
404 opened: jni::sys::jboolean,
405 detail: jni::objects::JString<'local>,
406) {
407 let _ = unowned_env
408 .with_env(|env| -> jni::errors::Result<()> {
409 let detail_str = format!("{detail}");
410 let s = sink();
411 let ok = opened != jni::sys::JNI_FALSE;
412 s.opened.store(ok, Ordering::Relaxed);
413 if let Ok(mut e) = s.last_error.lock() {
414 if ok {
415 e.clear();
416 } else {
417 *e = detail_str.clone();
418 }
419 }
420 if ok {
421 log::info!("android camera: opened ({detail_str})");
422 } else {
423 log::warn!("android camera: not opened ({detail_str})");
424 }
425 Ok(())
426 })
427 .into_outcome();
428}