rickub
nandi/jolt-nativepublic⑂ Fork 0
⑂ c4f56b0a96ecb29336e2cf16c522b794ae62ad3f
Branches
main View all branches → View all tags →
Commits
⬇ Clone ▾
git clone https://git.rickub.com/nandi/jolt-native.git
git clone ssh://git@rickub.com/nandi/jolt-native.git

Host key fingerprint (ed25519): SHA256:iycHnxEyq0Q7uyVpB7JlznP0G7JrTPXLYRcAU5CSLhc — verify it before your first connect.

android_camera.rs · 413 lines · 13.4 KBRust Blame HistoryRaw
Let the media plane cross to the phone, camera and all fd0e21a nandi 18d ago1//! Android Camera2 → MoQ video publish bridge.
2//!
3//! Java `CameraCapture` (in APK `classes.dex`) opens Camera2 / ImageReader and
4//! calls native NV12 push methods implemented here. Frames land in a
5//! latest-only [`PushCameraSource`] that implements iroh-live's [`VideoSource`].
6//!
7//! JNI note: never resolve `CameraCapture` with `Env::find_class` from a
8//! native worker thread — that uses the system ClassLoader and misses APK
9//! classes. Use [`android_jni::load_app_class`] (Activity ClassLoader).
10//!
11//! The `JavaVM` and the Activity come from [`crate::android_jni`], which the
12//! glue fills in before a call can start — see the note there for why they
13//! arrive by hand rather than out of `ndk_context`.
14
15use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
16use std::sync::{Arc, Mutex, OnceLock};
17use std::time::Duration;
18
19use anyhow::{anyhow, Context, Result};
20use iroh_live::media::{
21 format::{Nv12Planes, PixelFormat, VideoFormat, VideoFrame},
22 traits::VideoSource,
23};
24
25use crate::android_jni;
26
27/// Java binary name for the Camera2 helper in APK `classes.dex`.
28const CAMERA_CAPTURE_CLASS: &str = "uk.nandi.frq.CameraCapture";
29
30/// What every camera call says when the glue never handed the handles over —
31/// an APK packaging `libjoltmoq.so` but not calling `joltmoq_android_init`.
32const NOT_INITIALISED: &str = "joltmoq_android_init has not run";
33
34/// Shared sink written by JNI and read by the encoder thread.
35struct FrameSink {
36 pending: Mutex<Option<VideoFrame>>,
37 format: Mutex<VideoFormat>,
38 /// Set true after Java reports the capture session is live.
39 opened: AtomicBool,
40 /// Last open error detail (empty when ok / idle).
41 last_error: Mutex<String>,
42 frames_pushed: AtomicU64,
43}
44
45impl FrameSink {
46 fn new(width: u32, height: u32) -> Self {
47 Self {
48 pending: Mutex::new(None),
49 format: Mutex::new(VideoFormat {
50 // Encoders that receive FrameData::Nv12 ignore this field;
51 // keep Rgba as the VideoFormat default (rusty-codecs has no Nv12 variant).
52 pixel_format: PixelFormat::Rgba,
53 dimensions: [width, height],
54 }),
55 opened: AtomicBool::new(false),
56 last_error: Mutex::new(String::new()),
57 frames_pushed: AtomicU64::new(0),
58 }
59 }
60}
61
62static SINK: OnceLock<Arc<FrameSink>> = OnceLock::new();
63
64fn sink() -> Arc<FrameSink> {
65 SINK.get_or_init(|| Arc::new(FrameSink::new(640, 480)))
66 .clone()
67}
68
69/// Latest-frame-only [`VideoSource`] fed by Camera2 JNI callbacks.
70pub struct PushCameraSource {
71 sink: Arc<FrameSink>,
72 name: String,
73}
74
75impl PushCameraSource {
76 pub fn new(label: impl Into<String>) -> Self {
77 Self {
78 sink: sink(),
79 name: label.into(),
80 }
81 }
82}
83
84impl VideoSource for PushCameraSource {
85 fn name(&self) -> &str {
86 &self.name
87 }
88
89 fn format(&self) -> VideoFormat {
90 self.sink
91 .format
92 .lock()
93 .map(|g| g.clone())
94 .unwrap_or(VideoFormat {
95 pixel_format: PixelFormat::Rgba,
96 dimensions: [640, 480],
97 })
98 }
99
100 fn pop_frame(&mut self) -> Result<Option<VideoFrame>> {
101 Ok(self.sink.pending.lock().ok().and_then(|mut g| g.take()))
102 }
103
104 fn start(&mut self) -> Result<()> {
105 Ok(())
106 }
107
108 fn stop(&mut self) -> Result<()> {
109 if let Ok(mut g) = self.sink.pending.lock() {
110 *g = None;
111 }
112 Ok(())
113 }
114}
115
116/// Enumerate cameras for the device picker (front first).
117/// Returns `(id, label)` pairs.
118pub fn list_cameras() -> Vec<(String, String)> {
119 match list_cameras_jni() {
120 Ok(list) => list,
121 Err(e) => {
122 // Warn: empty list makes AV report "camera unavailable" permanently.
123 log::warn!("android camera list: {e}");
124 Vec::new()
125 }
126 }
127}
128
129fn list_cameras_jni() -> Result<Vec<(String, String)>> {
130 let vm = android_jni::vm().context(NOT_INITIALISED)?;
131 let activity_ptr = android_jni::activity().context(NOT_INITIALISED)?;
132
133 use jni::objects::{JObject, JObjectArray, JString, JValue};
134 use jni::refs::Global;
135 use jni::{jni_sig, jni_str};
136
137 let mut out: Option<Result<Vec<(String, String)>>> = None;
138 vm.attach_current_thread(|env| -> jni::errors::Result<()> {
139 let activity = unsafe { env.as_cast_raw::<Global<JObject>>(&activity_ptr)? };
140 let cls = match android_jni::load_app_class(env, activity.as_ref(), CAMERA_CAPTURE_CLASS) {
141 Ok(c) => c,
142 Err(e) => {
143 out = Some(Err(anyhow!("CameraCapture class: {e}")));
144 return Ok(());
145 }
146 };
147 let arr_obj = env
148 .call_static_method(
149 &cls,
150 jni_str!("listCameras"),
151 jni_sig!((android.content.Context) -> [java.lang.String]),
152 &[JValue::Object(activity.as_ref())],
153 )?
154 .l()?;
155 let arr = env.cast_local::<JObjectArray>(arr_obj)?;
156 let n = arr.len(env)?;
157 let mut list = Vec::with_capacity(n);
158 for i in 0..n {
159 let obj = arr.get_element(env, i)?;
160 if obj.is_null() {
161 continue;
162 }
163 let jstr = env.cast_local::<JString>(obj)?;
164 let s = format!("{jstr}");
165 let (id, name) = match s.split_once('\t') {
166 Some((id, name)) => (id.to_string(), name.to_string()),
167 None => (s.clone(), s),
168 };
169 list.push((id, name));
170 }
171 out = Some(Ok(list));
172 Ok(())
173 })
174 .map_err(|e| anyhow!("list cameras JNI: {e}"))?;
175 out.unwrap_or_else(|| Err(anyhow!("list cameras JNI: no result")))
176}
177
178/// Start Camera2 capture. Returns when the open request is dispatched (session
179/// readiness is async — wait with [`wait_until_opened`]).
180pub fn start_capture(camera_id: Option<&str>) -> Result<()> {
181 let s = sink();
182 s.opened.store(false, Ordering::Relaxed);
183 s.frames_pushed.store(0, Ordering::Relaxed);
184 if let Ok(mut e) = s.last_error.lock() {
185 e.clear();
186 }
187 if let Ok(mut p) = s.pending.lock() {
188 *p = None;
189 }
190
191 let vm = android_jni::vm().context(NOT_INITIALISED)?;
192 let activity_ptr = android_jni::activity().context(NOT_INITIALISED)?;
193
194 use jni::objects::{JObject, JValue};
195 use jni::refs::Global;
196 use jni::{jni_sig, jni_str};
197
198 let id = camera_id.unwrap_or("").to_string();
199 let mut start_err: Option<anyhow::Error> = None;
200 vm.attach_current_thread(|env| -> jni::errors::Result<()> {
201 let activity = unsafe { env.as_cast_raw::<Global<JObject>>(&activity_ptr)? };
202 let cls = match android_jni::load_app_class(env, activity.as_ref(), CAMERA_CAPTURE_CLASS) {
203 Ok(c) => c,
204 Err(e) => {
205 start_err = Some(anyhow!("CameraCapture class: {e}"));
206 return Ok(());
207 }
208 };
209 let jid = env.new_string(&id)?;
210 env.call_static_method(
211 &cls,
212 jni_str!("start"),
213 jni_sig!((android.app.Activity, java.lang.String) -> void),
214 &[JValue::Object(activity.as_ref()), JValue::Object(&jid)],
215 )?;
216 Ok(())
217 })
218 .map_err(|e| anyhow!("CameraCapture.start: {e}"))?;
219 if let Some(e) = start_err {
220 return Err(e).context("CameraCapture.start");
221 }
222 Ok(())
223}
224
225/// Block briefly until Java reports the capture session is live.
226pub fn wait_until_opened(timeout: Duration) -> Result<()> {
227 let s = sink();
228 let deadline = std::time::Instant::now() + timeout;
229 while std::time::Instant::now() < deadline {
230 if s.opened.load(Ordering::Relaxed) {
231 return Ok(());
232 }
233 let err = s
234 .last_error
235 .lock()
236 .ok()
237 .map(|g| g.clone())
238 .unwrap_or_default();
239 if !err.is_empty() {
240 return Err(anyhow!("camera open failed: {err}"));
241 }
242 std::thread::sleep(Duration::from_millis(20));
243 }
244 let err = s
245 .last_error
246 .lock()
247 .ok()
248 .map(|g| g.clone())
249 .unwrap_or_default();
250 if s.opened.load(Ordering::Relaxed) {
251 Ok(())
252 } else if !err.is_empty() {
253 Err(anyhow!("camera open failed: {err}"))
254 } else {
255 Err(anyhow!("camera open timed out"))
256 }
257}
258
259/// Stop Camera2 capture (idempotent).
260pub fn stop_capture() {
261 let (Some(vm), Some(activity_ptr)) = (android_jni::vm(), android_jni::activity()) else {
262 return;
263 };
264
265 use jni::objects::{JObject, JValue};
266 use jni::refs::Global;
267 use jni::{jni_sig, jni_str};
268
269 let _ = vm.attach_current_thread(|env| -> jni::errors::Result<()> {
270 let activity = unsafe { env.as_cast_raw::<Global<JObject>>(&activity_ptr)? };
271 let cls = match android_jni::load_app_class(env, activity.as_ref(), CAMERA_CAPTURE_CLASS) {
272 Ok(c) => c,
273 Err(e) => {
274 log::warn!("android camera stop: CameraCapture class: {e}");
275 return Ok(());
276 }
277 };
278 env.call_static_method(
279 &cls,
280 jni_str!("stop"),
281 jni_sig!((android.app.Activity) -> void),
282 &[JValue::Object(activity.as_ref())],
283 )?;
284 Ok(())
285 });
286 let s = sink();
287 s.opened.store(false, Ordering::Relaxed);
288 if let Ok(mut p) = s.pending.lock() {
289 *p = None;
290 };
291}
292
293/// RAII guard that stops Camera2 when the media session ends.
294pub struct CameraCaptureGuard;
295
296impl Drop for CameraCaptureGuard {
297 fn drop(&mut self) {
298 stop_capture();
299 log::info!("android camera: capture stopped (guard drop)");
300 }
301}
302
303// ── JNI callbacks from CameraCapture.java ─────────────────────────────────
304
305#[unsafe(no_mangle)]
306pub extern "system" fn Java_uk_nandi_frq_CameraCapture_onNv12Frame<'local>(
307 mut unowned_env: jni::JNIEnv<'local>,
308 _class: jni::objects::JClass<'local>,
309 y_data: jni::objects::JByteArray<'local>,
310 uv_data: jni::objects::JByteArray<'local>,
311 width: jni::sys::jint,
312 height: jni::sys::jint,
313 y_stride: jni::sys::jint,
314 uv_stride: jni::sys::jint,
315 rotation_degrees: jni::sys::jint,
316) {
317 if width <= 0 || height <= 0 {
318 return;
319 }
320 let _ = unowned_env
321 .with_env(|env| -> jni::errors::Result<()> {
322 let y_bytes = env.convert_byte_array(&y_data)?;
323 let uv_bytes = env.convert_byte_array(&uv_data)?;
324 let src_w = width as u32;
325 let src_h = height as u32;
326 let rot = if rotation_degrees < 0 {
327 0
328 } else {
329 rotation_degrees as u32
330 };
331 // Apply sensor/display orientation so published + preview frames
332 // are upright (Camera2 ImageReader buffers are sensor-oriented).
333 let (y_bytes, uv_bytes, w, h, y_str, uv_str) =
334 match crate::nv12_orient::orient_nv12(
335 &y_bytes,
336 &uv_bytes,
337 src_w,
338 src_h,
339 y_stride as u32,
340 uv_stride as u32,
341 rot,
342 ) {
343 Some((y, uv, w, h)) => (y, uv, w, h, w, w),
344 None => (
345 y_bytes,
346 uv_bytes,
347 src_w,
348 src_h,
349 y_stride as u32,
350 uv_stride as u32,
351 ),
352 };
353 let frame = VideoFrame::new_nv12(
354 Nv12Planes {
355 y_data: y_bytes,
356 y_stride: y_str,
357 uv_data: uv_bytes,
358 uv_stride: uv_str,
359 width: w,
360 height: h,
361 },
362 Duration::ZERO,
363 );
364 let s = sink();
365 if let Ok(mut fmt) = s.format.lock() {
366 if fmt.dimensions != [w, h] {
367 *fmt = VideoFormat {
368 pixel_format: PixelFormat::Rgba,
369 dimensions: [w, h],
370 };
371 }
372 }
373 if let Ok(mut pending) = s.pending.lock() {
374 *pending = Some(frame);
375 }
376 let n = s.frames_pushed.fetch_add(1, Ordering::Relaxed);
377 if n == 0 {
378 log::info!("android camera: first NV12 frame {w}x{h} (src {src_w}x{src_h} rot={rot})");
379 }
380 Ok(())
381 })
382 .into_outcome();
383}
384
385#[unsafe(no_mangle)]
386pub extern "system" fn Java_uk_nandi_frq_CameraCapture_onCameraState<'local>(
387 mut unowned_env: jni::JNIEnv<'local>,
388 _class: jni::objects::JClass<'local>,
389 opened: jni::sys::jboolean,
390 detail: jni::objects::JString<'local>,
391) {
392 let _ = unowned_env
393 .with_env(|env| -> jni::errors::Result<()> {
394 let detail_str = format!("{detail}");
395 let s = sink();
396 let ok = opened != jni::sys::JNI_FALSE;
397 s.opened.store(ok, Ordering::Relaxed);
398 if let Ok(mut e) = s.last_error.lock() {
399 if ok {
400 e.clear();
401 } else {
402 *e = detail_str.clone();
403 }
404 }
405 if ok {
406 log::info!("android camera: opened ({detail_str})");
407 } else {
408 log::warn!("android camera: not opened ({detail_str})");
409 }
410 Ok(())
411 })
412 .into_outcome();
413}