diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 1896e82..c7bbb93 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -153,11 +153,16 @@ abstract class CompileSlangShadersTask : DefaultTask() { val compileProcess = ProcessBuilder( "slangc", file.absolutePath, "-target", "spirv", "-o", outputFile.absolutePath - ).inheritIO().start() + ).redirectErrorStream(true).start() + + val output = compileProcess.inputStream.bufferedReader().readText() + if (output.isNotBlank()) { + logger.lifecycle(output.trim()) + } val exitCode = compileProcess.waitFor() if (exitCode != 0) { - throw GradleException("slangc failed compiling ${file.name} with exit code $exitCode") + throw GradleException("slangc failed compiling ${file.name} with exit code $exitCode, see log for details") } } } diff --git a/app/src/generated/assets/shaders/egui.spv b/app/src/generated/assets/shaders/egui.spv new file mode 100644 index 0000000..d7d121b Binary files /dev/null and b/app/src/generated/assets/shaders/egui.spv differ diff --git a/app/src/generated/assets/shaders/gltf.spv b/app/src/generated/assets/shaders/gltf.spv deleted file mode 100644 index 4b80283..0000000 Binary files a/app/src/generated/assets/shaders/gltf.spv and /dev/null differ diff --git a/app/src/generated/assets/shaders/gltf_unlit_translucent.spv b/app/src/generated/assets/shaders/gltf_unlit_translucent.spv index b6940f0..467c423 100644 Binary files a/app/src/generated/assets/shaders/gltf_unlit_translucent.spv and b/app/src/generated/assets/shaders/gltf_unlit_translucent.spv differ diff --git a/app/src/main/assets/fonts/symbols.ttf b/app/src/main/assets/fonts/symbols.ttf new file mode 100644 index 0000000..440dd05 Binary files /dev/null and b/app/src/main/assets/fonts/symbols.ttf differ diff --git a/app/src/main/assets/meshes/keyboard.glb b/app/src/main/assets/meshes/keyboard.glb new file mode 100644 index 0000000..6bd0c33 Binary files /dev/null and b/app/src/main/assets/meshes/keyboard.glb differ diff --git a/app/src/main/assets/shaders/egui.slang b/app/src/main/assets/shaders/egui.slang new file mode 100644 index 0000000..4f1aa2d --- /dev/null +++ b/app/src/main/assets/shaders/egui.slang @@ -0,0 +1,54 @@ +struct VertexInput { + float2 pos : POSITION; + float2 uv : TEXCOORD0; + uint color : COLOR; +} + +struct VertexOutput { + float4 position : SV_Position; + float4 color : COLOR; + float2 uv : TEXCOORD0; +} + +struct PushConstants { + float2 screen_size; +} + +[[vk::binding(0, 0)]] +Sampler2D font_sampler_lle; + +float4 linear_from_srgba(float4 srgba) { + bool4 cutoff = srgba < 0.04045; + float4 higher = pow((srgba + 0.055) / 1.055, 2.4); + float4 lower = srgba / 12.92; + + return lerp(higher, lower, float4(cutoff)); +} + +float4 srgba_from_linear(float4 linear) { + bool4 cutoff = linear < 0.0031308; + float4 higher = 1.055 * pow(linear, 1.0 / 2.4) - 0.055; + float4 lower = linear * 12.92; + + return lerp(higher, lower, float4(cutoff)); +} + +[shader("vertex")] +VertexOutput vertex_main( + uniform PushConstants pc, + VertexInput input +) { + VertexOutput output; + float2 pos = (input.pos / pc.screen_size) * 2.0 - 1.0; + output.position = float4(pos, 0.0, 1.0); + + output.color = linear_from_srgba(unpackUnorm4x8ToFloat(input.color)); + output.uv = input.uv; + + return output; +} + +[shader("fragment")] +float4 fragment_main(VertexOutput input) { + return srgba_from_linear(input.color * font_sampler_lle.Sample(input.uv)); +} \ No newline at end of file diff --git a/app/src/main/assets/shaders/gltf_unlit_translucent.slang b/app/src/main/assets/shaders/gltf_unlit_translucent.slang index d7ad744..be8ce98 100644 --- a/app/src/main/assets/shaders/gltf_unlit_translucent.slang +++ b/app/src/main/assets/shaders/gltf_unlit_translucent.slang @@ -119,7 +119,6 @@ float4 fragment_main(VertexStageOutput input) : SV_Target { tex_albedo = float4(0.0); } - clip(tex_albedo.a - 0.1); albedo *= tex_albedo; } diff --git a/app/src/main/java/com/qcxr/questcraft/JniBridge.java b/app/src/main/java/com/qcxr/questcraft/JniBridge.java index 121a06e..1c23ee3 100644 --- a/app/src/main/java/com/qcxr/questcraft/JniBridge.java +++ b/app/src/main/java/com/qcxr/questcraft/JniBridge.java @@ -2,6 +2,7 @@ import android.content.res.AssetManager; import android.view.Surface; +import com.qcxr.questcraft.ui.XrTextInputBridge; @SuppressWarnings("unused") public class JniBridge { @@ -9,9 +10,11 @@ public class JniBridge { System.loadLibrary("qcxr"); } + public static native void setSkinImage(byte[] imageBytes, boolean slim); public static native void start(MainActivity activity, AssetManager assetManager); public static native void stop(); - public static native void setSkinImage(byte[] imageBytes, boolean slim); + public static native void showKeyboard(); + public static native void hideKeyboard(); public static void setVulkanSurface(Surface surface, int width, int height) { MainActivity.instance().ifPresent(me -> me.setVulkanSurface(surface, width, height)); @@ -28,4 +31,12 @@ public static void requestUiRender() { public static void processPointerEvent(int pointerId, int action, float normX, float normY) { MainActivity.instance().ifPresent(me -> me.processPointerEvent(pointerId, action, normX, normY)); } + + public static void sendText(String text) { + XrTextInputBridge.sendText(text); + } + + public static void deleteCharacter() { + XrTextInputBridge.deleteCharacter(); + } } diff --git a/app/src/main/java/com/qcxr/questcraft/MainActivity.java b/app/src/main/java/com/qcxr/questcraft/MainActivity.java index 66c54a3..7c71aec 100644 --- a/app/src/main/java/com/qcxr/questcraft/MainActivity.java +++ b/app/src/main/java/com/qcxr/questcraft/MainActivity.java @@ -1,10 +1,8 @@ package com.qcxr.questcraft; -import android.annotation.SuppressLint; import android.os.Bundle; import android.os.Handler; import android.os.Looper; -import android.util.DisplayMetrics; import android.view.Surface; import android.view.View; import android.view.ViewGroup; @@ -38,7 +36,6 @@ public static Optional instance() { @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - //setDensity(); weakMe = new WeakReference<>(this); uiThreadHandler = new Handler(Looper.getMainLooper()); xrActivityInput = new XRActivityInput(uiThreadHandler); @@ -76,16 +73,6 @@ private void deviceCodeCallback(DeviceCode res) { } } - private void setDensity() { - DisplayMetrics metrics = getResources().getDisplayMetrics(); - float density = 0.75f; - float densityMultiplier = 160; - metrics.density = density; - metrics.ydpi = metrics.xdpi = densityMultiplier * density; - metrics.densityDpi = (int)metrics.xdpi; - getResources().updateConfiguration(null, null); - } - public void setVulkanSurface(Surface surface, int width, int height) { runOnUiThread(() -> { this.nativeSurface = new NativeSurface(this); diff --git a/app/src/main/java/com/qcxr/questcraft/ui/XrTextInputBridge.kt b/app/src/main/java/com/qcxr/questcraft/ui/XrTextInputBridge.kt new file mode 100644 index 0000000..6bdabdf --- /dev/null +++ b/app/src/main/java/com/qcxr/questcraft/ui/XrTextInputBridge.kt @@ -0,0 +1,57 @@ +package com.qcxr.questcraft.ui + +import android.os.Handler +import android.os.Looper +import androidx.compose.ui.text.TextRange +import androidx.compose.ui.text.input.TextFieldValue +import com.qcxr.questcraft.JniBridge + +object XrTextInputBridge { + private val mainHandler = Handler(Looper.getMainLooper()) + + private data class ActiveField( + val getValue: () -> TextFieldValue, + val setValue: (TextFieldValue) -> Unit, + ) + + @Volatile private var active: ActiveField? = null + + internal fun register(getValue: () -> TextFieldValue, setValue: (TextFieldValue) -> Unit) { + active = ActiveField(getValue, setValue) + JniBridge.showKeyboard() + } + + internal fun unregister(getValue: () -> TextFieldValue) { + if (active?.getValue === getValue) { + active = null + JniBridge.hideKeyboard() + } + } + + @JvmStatic + fun sendText(text: String) { + mainHandler.post { + val field = active ?: return@post + val v = field.getValue() + val newText = v.text.replaceRange(v.selection.start, v.selection.end, text) + val newCursor = v.selection.start + text.length + field.setValue(TextFieldValue(newText, TextRange(newCursor))) + } + } + + @JvmStatic + fun deleteCharacter() { + mainHandler.post { + val field = active ?: return@post + val v = field.getValue() + if (v.selection.start != v.selection.end) { + val newText = v.text.removeRange(v.selection.start, v.selection.end) + field.setValue(TextFieldValue(newText, TextRange(v.selection.start))) + } else if (v.selection.start > 0) { + val cut = v.selection.start - 1 + val newText = v.text.removeRange(cut, v.selection.start) + field.setValue(TextFieldValue(newText, TextRange(cut))) + } + } + } +} \ No newline at end of file diff --git a/app/src/main/java/com/qcxr/questcraft/ui/components/AddInstanceOverlay.kt b/app/src/main/java/com/qcxr/questcraft/ui/components/AddInstanceOverlay.kt index 2b7ff3b..bbe89b7 100644 --- a/app/src/main/java/com/qcxr/questcraft/ui/components/AddInstanceOverlay.kt +++ b/app/src/main/java/com/qcxr/questcraft/ui/components/AddInstanceOverlay.kt @@ -7,7 +7,6 @@ import androidx.compose.foundation.layout.* import androidx.compose.foundation.shape.RoundedCornerShape import androidx.compose.material3.Button import androidx.compose.material3.ButtonDefaults -import androidx.compose.material3.OutlinedTextField import androidx.compose.material3.OutlinedTextFieldDefaults import androidx.compose.material3.Surface import androidx.compose.material3.Text @@ -17,6 +16,7 @@ import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.font.FontWeight +import androidx.compose.ui.text.input.TextFieldValue import androidx.compose.ui.tooling.preview.Preview import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp @@ -28,7 +28,7 @@ fun AddInstanceOverlay( onDismiss: () -> Unit, onCreate: (name: String, version: String, loader: String) -> Unit ) { - var instanceName by remember { mutableStateOf("") } + var instanceName by remember { mutableStateOf(TextFieldValue(text = "")) } var selectedVersion by remember { mutableStateOf("1.20.1") } var selectedLoader by remember { mutableStateOf("Fabric") } @@ -62,7 +62,7 @@ fun AddInstanceOverlay( Text(text = stringResource(R.string.instance_name), color = TextSecondary, fontSize = 12.sp) Spacer(modifier = Modifier.height(8.dp)) - OutlinedTextField( + XrOutlinedTextField( value = instanceName, onValueChange = { instanceName = it }, modifier = Modifier.fillMaxWidth(), @@ -128,10 +128,10 @@ fun AddInstanceOverlay( Spacer(modifier = Modifier.width(8.dp)) Button( - onClick = { onCreate(instanceName, selectedVersion, selectedLoader) }, + onClick = { onCreate(instanceName.text, selectedVersion, selectedLoader) }, colors = ButtonDefaults.buttonColors(containerColor = AccentGreen), shape = RoundedCornerShape(4.dp), - enabled = instanceName.isNotBlank() + enabled = instanceName.text.isNotBlank() ) { Text(text = stringResource(R.string.create), color = Color.White) } diff --git a/app/src/main/java/com/qcxr/questcraft/ui/components/XrTextField.kt b/app/src/main/java/com/qcxr/questcraft/ui/components/XrTextField.kt new file mode 100644 index 0000000..558f43d --- /dev/null +++ b/app/src/main/java/com/qcxr/questcraft/ui/components/XrTextField.kt @@ -0,0 +1,87 @@ +package com.qcxr.questcraft.ui.components + +import androidx.compose.foundation.interaction.MutableInteractionSource +import androidx.compose.foundation.text.KeyboardActions +import androidx.compose.foundation.text.KeyboardOptions +import androidx.compose.material3.LocalTextStyle +import androidx.compose.material3.OutlinedTextField +import androidx.compose.material3.OutlinedTextFieldDefaults +import androidx.compose.material3.TextFieldColors +import androidx.compose.runtime.Composable +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.ui.Modifier +import androidx.compose.ui.focus.onFocusChanged +import androidx.compose.ui.graphics.Shape +import androidx.compose.ui.platform.LocalSoftwareKeyboardController +import androidx.compose.ui.text.TextStyle +import androidx.compose.ui.text.input.TextFieldValue +import androidx.compose.ui.text.input.VisualTransformation +import com.qcxr.questcraft.ui.XrTextInputBridge + +@Composable +fun XrOutlinedTextField( + value: TextFieldValue, + onValueChange: (TextFieldValue) -> Unit, + modifier: Modifier = Modifier, + enabled: Boolean = true, + readOnly: Boolean = false, + textStyle: TextStyle = LocalTextStyle.current, + label: @Composable (() -> Unit)? = null, + placeholder: @Composable (() -> Unit)? = null, + leadingIcon: @Composable (() -> Unit)? = null, + trailingIcon: @Composable (() -> Unit)? = null, + prefix: @Composable (() -> Unit)? = null, + suffix: @Composable (() -> Unit)? = null, + supportingText: @Composable (() -> Unit)? = null, + isError: Boolean = false, + visualTransformation: VisualTransformation = VisualTransformation.None, + keyboardOptions: KeyboardOptions = KeyboardOptions.Default, + keyboardActions: KeyboardActions = KeyboardActions.Default, + singleLine: Boolean = false, + maxLines: Int = if (singleLine) 1 else Int.MAX_VALUE, + minLines: Int = 1, + interactionSource: MutableInteractionSource? = null, + shape: Shape = OutlinedTextFieldDefaults.shape, + colors: TextFieldColors = OutlinedTextFieldDefaults.colors(), +) { + val keyboardController = LocalSoftwareKeyboardController.current + val currentValue = rememberUpdatedState(value) + val currentOnValueChange = rememberUpdatedState(onValueChange) + + val getValue = remember { { currentValue.value } } + val setValue = remember { { v: TextFieldValue -> currentOnValueChange.value(v) } } + + OutlinedTextField( + value = value, + onValueChange = onValueChange, + textStyle = textStyle, + modifier = modifier.onFocusChanged { focus -> + if (focus.isFocused) { + keyboardController?.hide() + XrTextInputBridge.register(getValue, setValue) + } else { + XrTextInputBridge.unregister(getValue) + } + }, + enabled = enabled, + readOnly = readOnly, + label = label, + placeholder = placeholder, + leadingIcon = leadingIcon, + trailingIcon = trailingIcon, + prefix = prefix, + suffix = suffix, + supportingText = supportingText, + isError = isError, + visualTransformation = visualTransformation, + keyboardOptions = keyboardOptions, + keyboardActions = keyboardActions, + singleLine = singleLine, + maxLines = maxLines, + minLines = minLines, + interactionSource = interactionSource, + shape = shape, + colors = colors, + ) +} \ No newline at end of file diff --git a/app/src/main/rust/cargo.toml b/app/src/main/rust/cargo.toml index 54b6246..ed62642 100644 --- a/app/src/main/rust/cargo.toml +++ b/app/src/main/rust/cargo.toml @@ -21,7 +21,7 @@ bytemuck = { version = "1.25", features = ["derive"] } glam = { version = "0.33", features = ["bytemuck", "mint"] } mint = "0.5" openxr = { git = "https://github.com/Ralith/openxrs.git", rev = "294809e4fe7f6f9729c80526f5320d7d70408984", default-features = false, features = ["mint"] } -vk-graph = { version = "0.14.5" } +vk-graph = { version = "0.14.7" } egui = { version = "0.35.0", features = [ "bytemuck" ] } gltf = { version = "1.4", features = [ "extras", "names" ] } diff --git a/app/src/main/rust/src/app.rs b/app/src/main/rust/src/app.rs index 3cea0e4..56a5b5a 100644 --- a/app/src/main/rust/src/app.rs +++ b/app/src/main/rust/src/app.rs @@ -8,14 +8,17 @@ use { crate::{ jni_state::{self, JniContext}, render::{ - renderer::{self, PollError, Renderer, WaitError}, + renderer::{self, PollError, Renderer, WaitError, RenderingError, XrCameraUBO}, Swapchain }, stage::Stage, scene::scene::{Scene, Skin, SkinType}, input::{InputState, ExtractedInputs, Hand}, instance::XrInstance, - surface::{Surface, SurfaceManager, SurfaceTexture} + surface::{AndroidSurface, SurfaceManager, SurfaceTexture}, + egui::Egui, + keyboard::Keyboard, + geometry::RaycastHit }, jni::{ Env @@ -23,17 +26,18 @@ use { glam::{Mat4, Quat, Vec2, Vec3}, ndk::asset::AssetManager, ndk_sys::{AAssetManager, AMOTION_EVENT_ACTION_DOWN, AMOTION_EVENT_ACTION_MOVE, AMOTION_EVENT_ACTION_UP}, - openxr::{self as xr, EnvironmentBlendMode, FrameState, ViewConfigurationType}, + openxr::{self as xr, EnvironmentBlendMode, FrameState, ViewConfigurationType, Posef}, vk_graph::{ driver::{ - ash::vk::{self, BufferUsageFlags}, - buffer::Buffer, + ash::vk::{self, BufferUsageFlags, DeviceSize}, + buffer::{Buffer, BufferInfo}, image::{Image, ImageInfoBuilder}, }, pool::hash::HashPool, Graph, }, - log::info, + log::{info, error}, + egui::{remap}, }; pub struct XrSession { @@ -43,8 +47,17 @@ pub struct XrSession { pub frame_stream: xr::FrameStream, } +pub struct RecenterInfo { + pub pose_in_previous_space: Posef, + pub reference_space_type: xr::ReferenceSpaceType, +} + +pub enum PollResult { + Recenter(RecenterInfo), +} + impl XrSession { - pub fn poll(&mut self, instance: &mut XrInstance) -> Result<(), PollError> { + pub fn poll(&mut self, instance: &mut XrInstance) -> Result, PollError> { while let Some(event) = XrInstance::poll_event(instance).unwrap() { use xr::Event::*; match event { @@ -76,18 +89,35 @@ impl XrSession { EventsLost(e) => { log::error!("lost {} events", e.lost_event_count()); } + ReferenceSpaceChangePending(e) => { + return Ok(Some(PollResult::Recenter(RecenterInfo { + pose_in_previous_space: e.pose_in_previous_space(), + reference_space_type: e.reference_space_type(), + }))) + } _ => {} } }; - Ok(()) + Ok(None) } pub fn wait_frame(&mut self) -> Result { - let xr_frame_state = self.frame_wait.wait().map_err(|_err| { - WaitError::DriverError - })?; + let xr_frame_state = match self.frame_wait.wait() { + Ok(state) => state, + Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => { + self.running = false; + return Err(WaitError::Sleeping); + } + Err(err) => { + error!("Failed to wait for frame {:?}", err); + return Err(WaitError::DriverError); + } + }; - self.frame_stream.begin().unwrap(); + if let Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) = self.frame_stream.begin() { + self.running = false; + return Err(WaitError::Sleeping); + } if !xr_frame_state.should_render { self.frame_stream.end( @@ -107,7 +137,8 @@ pub struct XrContext { pub queue_family_index: u32, // this is like primarily graphics, but it makes sense to force it into the "xr context" because the xr context is inherently tied to it pub session: XrSession, pub swapchain: Swapchain, - pub stage: xr::Space, + pub stage_space: xr::Space, + pub view_space: xr::Space, } impl XrContext { @@ -123,7 +154,10 @@ impl XrContext { let swapchain = Swapchain::new(&instance, &session); let stage = session - .create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY) + .create_reference_space(xr::ReferenceSpaceType::STAGE, Posef::IDENTITY) + .unwrap(); + let view = session + .create_reference_space(xr::ReferenceSpaceType::VIEW, Posef::IDENTITY) .unwrap(); XrContext { @@ -133,10 +167,11 @@ impl XrContext { session, frame_wait, frame_stream, - running: true, + running: false, }, swapchain, - stage, + stage_space: stage, + view_space: view, } } } @@ -162,23 +197,58 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu let mut context = XrContext::new(Arc::clone(&ctx)); let mut renderer = Renderer::new(&context, internal_files_directory.as_path()); + let mut egui = Egui::new(&context.instance.device, &asset_manager); + { + let mut symbol_asset = asset_manager.open(c"fonts/symbols.ttf") + .expect("Failed to load symbols font"); + let font_bytes = symbol_asset.buffer().unwrap().to_vec(); + + let mut fonts = egui::FontDefinitions::default(); + fonts.font_data.insert( + "keyboard-symbols".to_owned(), + Arc::new(egui::FontData::from_owned(font_bytes)), + ); + + fonts.families + .get_mut(&egui::FontFamily::Proportional) + .unwrap() + .push("keyboard-symbols".to_owned()); + + egui.ctx.set_fonts(fonts); + } let input = InputState::new(&context.instance, &context.session.session); - let mut scene = Scene::load(&context.instance.device, &asset_manager); + + let mut mapped_camera_buffers = Vec::new(); + let mut camera_buffers = Vec::new(); + (0..context.swapchain.enumerate_images().unwrap().len()).for_each(|_| { + let mut buffer = Buffer::create(&context.instance.device, BufferInfo::builder() + .host_writable(true) + .size(size_of::() as DeviceSize) + .usage(BufferUsageFlags::UNIFORM_BUFFER)) + .expect("Failed to allocate camera buffer"); + mapped_camera_buffers.push(buffer.mapped_slice_mut().as_mut_ptr() as *mut XrCameraUBO); + camera_buffers.push(Arc::new(buffer)); + }); + + let mut scene = Scene::load(&context.instance.device, &camera_buffers, &asset_manager); + let mut keyboard = Keyboard::new(&context.instance.device, &camera_buffers, &asset_manager, ctx.clone(), vk::Extent2D{width: 1280, height: 512}); // if you're looking how to load assets n shit, it's in the scene ^^ let names: Vec<&str> = scene.assets.animated_asset.animation_names().collect(); info!("Animations: {:?}", names); let mut animator = scene.assets.animated_instance.create_animation_player("Idle", true).expect("Failed to get Idle animator"); - let mut surface = Surface::new(env, 1920, 1080); + let mut surface = AndroidSurface::new(env, 1920, 1080); ctx.set_surface(env, &surface); let surface_index = scene.surface_index.expect("Scene mesh must contain a node tagged as a surface! (see custom properties in Blender)"); let surface_node = &scene.assets.scene_instance.nodes[surface_index]; - let surface_mesh = &scene.assets.scene_asset.meshes[surface_node.mesh_index.expect("Scene node tagged as surface doesn't contain a mesh (what are we supposed to render the texture onto??)")].primitives[0]; let surface_transform = surface_node.global_transform; - let surface_manager = SurfaceManager::new(&asset_manager, &context.instance.device, &scene.assets.scene_asset, surface_mesh); + let surface_manager = SurfaceManager::new(&asset_manager, &context.instance.device, scene.assets.scene_asset.clone(), [ + surface_node.mesh_index.expect("Scene node tagged as surface doesn't contain a mesh (what are we supposed to render the texture onto??)"), + 0 + ]); let spawn = scene.spawn_point.unwrap_or(Mat4::IDENTITY); let (_, rotation, translation) = spawn.to_scale_rotation_translation(); @@ -188,6 +258,7 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu let mut last_surface_texture: Option> = None; let mut previous_inputs: Option = None; let mut primary_hand = Hand::Right; + let mut was_running = false; info!("Starting!!"); let mut last_frame_time = Instant::now(); @@ -196,13 +267,18 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu let delta_time = now.duration_since(last_frame_time).as_secs_f32(); last_frame_time = now; - context.session.poll(&mut context.instance).expect("Failed to poll session"); + let result = context.session.poll(&mut context.instance).expect("Failed to poll events"); + if !context.session.running { + was_running = false; + sleep(Duration::from_millis(10)); + continue; + } let active_frame = match renderer.begin_frame(&mut context) { Ok(frame) => frame, Err(err) => { match err { - renderer::RenderingError::Sleeping => { + RenderingError::Sleeping => { sleep(Duration::from_millis(100)); log::trace!("sleeping...") }, @@ -212,18 +288,21 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu } }; - old_surface_textures.retain(|texture: &Arc| { - Arc::strong_count(&texture.image) > 1 - }); - - ctx.request_ui_render(env); // hope to fuck this finishes before we need the texture + if context.session.running && !was_running { + was_running = true; + keyboard.adjust_position(&stage, context.view_space.locate(&context.stage_space, active_frame.predicted_display_time).unwrap().pose.position.y); + } + if let Some(PollResult::Recenter(_info)) = result { + keyboard.adjust_position(&stage, context.view_space.locate(&context.stage_space, active_frame.predicted_display_time).unwrap().pose.position.y); + } + keyboard.hidden = jni_state::KEYBOARD_HIDDEN.load(Ordering::Relaxed); - let inputs = input.extract(&context.session.session, &context.stage, active_frame.predicted_display_time); + let inputs = input.extract(&context.session.session, &context.stage_space, active_frame.predicted_display_time); let (_, views) = context.session.session.locate_views( ViewConfigurationType::PRIMARY_STEREO, active_frame.predicted_display_time, - &context.stage, + &context.stage_space, ).expect("Failed to locate tracking views"); let left_eye_rot: mint::Quaternion = views[0].pose.orientation.into(); @@ -240,6 +319,9 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu if let Some(surface_texture) = surface_texture { last_surface_texture = Some(Arc::new(surface_texture)); } + old_surface_textures.retain(|texture: &Arc| { + Arc::strong_count(&texture.image) > 1 + }); if let Some(data) = jni_state::PENDING_SKIN_IMAGE.lock().unwrap().take() { match image::load_from_memory(&data.png_bytes) { @@ -275,6 +357,14 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu } Err(err) => log::error!("Bad skin supplied from java-side: {:?}", err) } + + if let Some(ref skin) = *scene.assets.skin.read().unwrap() { + scene.assets.animated_instance.override_textures(&skin.texture, &camera_buffers); + scene.assets.left_controller_scene_instance.override_textures(&skin.texture, &camera_buffers); + scene.assets.slim_left_controller_scene_instance.override_textures(&skin.texture, &camera_buffers); + scene.assets.right_controller_scene_instance.override_textures(&skin.texture, &camera_buffers); + scene.assets.slim_right_controller_scene_instance.override_textures(&skin.texture, &camera_buffers); + } } animator.advance(delta_time, &scene.assets.animated_asset.animations[animator.clip_index]); @@ -303,7 +393,7 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu let hand_inputs = if primary_hand == Hand::Right { inputs.right - } else { + } else { inputs.left }; let previous_hand_inputs = previous_inputs.map(|inputs| { @@ -316,43 +406,60 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu let mut pointer_transform = None; if let Some(ref transform) = hand_inputs.matrix { - let transform = stage_to_world * transform * Mat4::from_translation(Vec3::new(0.0,0.0,0.127)); - let hit = surface_manager.raycast_uv(transform, surface_transform, -Vec3::Z); - - if let Some(hit) = hit { + let transform = stage_to_world * transform * Mat4::from_translation(Vec3::new(0.0, 0.0, 0.127)); + let primary_surface_raycast = surface_manager.raycast_uv(transform, surface_transform, -Vec3::Z); + let raycasts = [ + primary_surface_raycast, + if !keyboard.hidden { keyboard.surface_manager.raycast_uv(transform, keyboard.transform * keyboard.primitive_transform, -Vec3::Z) } else { None }, + ]; + let hit = RaycastHit::closest_hit(&raycasts); + + if let Some((hit, index)) = hit { let uv = hit.uv; + // 0.25 @ 0.5, 1 @ 6.5 + let scale = remap(hit.distance, 0.25f32..=6f32, 0.15f32..=1.0f32 ); pointer_transform = Some(Mat4::from_scale_rotation_translation( - Vec3::ONE, - Quat::from_rotation_arc(Vec3::Y, hit.world_normal.normalize()), - hit.world_position + Vec3::ONE * scale, + Quat::from_rotation_arc(Vec3::Y, hit.normal.normalize()), + hit.position + hit.normal * 0.01f32, )); if let Some(previous_hand_inputs) = previous_hand_inputs { - publish_inputs_for_pointer(env, &ctx, 0, previous_hand_inputs.click, hand_inputs.click, uv); + if index == 0 { + publish_inputs_for_pointer(env, &ctx, 0, previous_hand_inputs.click, hand_inputs.click, uv); + } else if !keyboard.hidden && index == 1 { + keyboard.publish_inputs(previous_hand_inputs.click, hand_inputs.click, uv); + } } + } else { + keyboard.publish_pointer_leave(); // todo: figure out *when* the pointer leaves } } - let payload = renderer::FramePayload { - view_matrices: [ + unsafe { + let mapped_camera_buffer = mapped_camera_buffers[renderer.frame_in_flight]; + (*mapped_camera_buffer).view_matrices = [ renderer::view_transform(views[0]) * world_to_stage, renderer::view_transform(views[1]) * world_to_stage, - ], - projection_matrices: [ + ]; + (*mapped_camera_buffer).projection_matrices = [ renderer::projection_transform(views[0]), renderer::projection_transform(views[1]), - ], + ]; + } + + let payload = renderer::FramePayload { + camera_buffer: &camera_buffers[renderer.frame_in_flight], xr_views: &views, }; let result = renderer.draw(&mut context, active_frame, payload, |graph, draw_payload| { scene.record(graph, draw_payload); - if let Some(ref skin) = *scene.assets.skin.read().unwrap() { - scene.assets.animated_instance.record_with_transform_override_texture(graph, draw_payload, &Mat4::IDENTITY, skin.texture.clone()); - } + scene.assets.animated_instance.record_with_transform(graph, draw_payload, &Mat4::IDENTITY); if let Some(ref last_surface_texture) = last_surface_texture { - surface_manager.record_with_transform(graph, last_surface_texture.image.clone(), draw_payload, surface_transform); + surface_manager.record_with_transform(graph, last_surface_texture.image.clone(), draw_payload, &surface_transform); } + keyboard.record(env, &mut egui, graph, draw_payload); if let Some(ref transform) = pointer_transform { scene.record_pointer(graph, draw_payload, transform); } @@ -386,6 +493,8 @@ pub fn main_loop(env: &mut Env<'_>, ctx: Arc, raw_asset_manager: *mu if let Some(ref last_surface_texture) = last_surface_texture { old_surface_textures.push(Arc::clone(last_surface_texture)); } + + ctx.request_ui_render(env); // hope to fuck this finishes before we need the texture } info!("Exiting..."); diff --git a/app/src/main/rust/src/egui/mod.rs b/app/src/main/rust/src/egui/mod.rs index 13413f0..c4428e4 100644 --- a/app/src/main/rust/src/egui/mod.rs +++ b/app/src/main/rust/src/egui/mod.rs @@ -11,6 +11,7 @@ use vk_graph::driver::image::{Image, ImageInfo}; use vk_graph::driver::shader::Shader; use vk_graph::driver::sync::AccessType; use vk_graph::{Graph, LoadOp, StoreOp}; +use vk_graph::driver::ash::vk::ShaderStageFlags; use vk_graph::node::AnyImageNode; use vk_graph::pool::hash::HashPool; use vk_graph::pool::{Lease, Pool}; @@ -51,9 +52,39 @@ impl Egui { [ Shader::builder() .entry_name("vertex_main") + .stage(ShaderStageFlags::VERTEX) + .vertex_input( + [ + vk::VertexInputBindingDescription { + binding: 0, + stride: size_of::() as u32, + input_rate: vk::VertexInputRate::VERTEX, + } + ], [ + vk::VertexInputAttributeDescription { + location: 0, + binding: 0, + format: vk::Format::R32G32_SFLOAT, + offset: 0, + }, + vk::VertexInputAttributeDescription { + location: 1, + binding: 0, + format: vk::Format::R32G32_SFLOAT, + offset: 8, + }, + vk::VertexInputAttributeDescription { + location: 2, + binding: 0, + format: vk::Format::R32_UINT, + offset: 16, + } + ] + ) .spirv(spv_bytes), Shader::builder() .entry_name("fragment_main") + .stage(ShaderStageFlags::FRAGMENT) .spirv(spv_bytes) ] ).expect("Failed to create egui pipeline") diff --git a/app/src/main/rust/src/geometry.rs b/app/src/main/rust/src/geometry.rs new file mode 100644 index 0000000..756346a --- /dev/null +++ b/app/src/main/rust/src/geometry.rs @@ -0,0 +1,98 @@ +use glam::{Vec2, Vec3}; +use crate::scene::gltf_model::Vertex; + +#[derive(Debug, PartialEq, Copy, Clone)] +pub struct RaycastHit { + pub uv: Vec2, + pub position: Vec3, + pub normal: Vec3, + pub distance: f32, +} + +impl RaycastHit { + pub fn closest_hit(hits: &[Option]) -> Option<(&RaycastHit, usize)> { + let hit = hits.iter() + .filter_map(|hit| hit.as_ref()) + .min_by(|a,b| a.distance.total_cmp(&b.distance)); + if hit == None { + return None + } + let index = hits.iter().position(|hit| hit.is_some()).unwrap(); + Some((hit.unwrap(), index)) + } +} + +// this uses the Moller-Trumbore intersection algorithm +// all inputs should be in the same space, the outputs will be in the same space +pub fn raycast_uv(vertices: &Vec, indices: &Vec, ray_origin: Vec3, ray_direction: Vec3) -> Option { + let mut closest_t = f32::INFINITY; + + let mut closest_local_pos = Vec3::ZERO; + let mut closest_local_normal = Vec3::ZERO; + let mut closest_uv = Vec2::ZERO; + let mut hit_found = false; + + for chunk in indices.chunks_exact(3) { + let v0 = &vertices[chunk[0] as usize]; + let v1 = &vertices[chunk[1] as usize]; + let v2 = &vertices[chunk[2] as usize]; + + let p0 = Vec3::from_array(v0.position); + let p1 = Vec3::from_array(v1.position); + let p2 = Vec3::from_array(v2.position); + + let edge1 = p1 - p0; + let edge2 = p2 - p0; + let h = ray_direction.cross(edge2); + let a = edge1.dot(h); + + if a.abs() < f32::EPSILON { continue; } + + let f = 1.0 / a; + let s = ray_origin - p0; + let u = f * s.dot(h); + + if u < 0.0 || u > 1.0 { continue; } + + let q = s.cross(edge1); + let v = f * ray_direction.dot(q); + + if v < 0.0 || u + v > 1.0 { continue; } + + if u + v > 1.0 { continue; } + + let t = f * edge2.dot(q); + + if t > f32::EPSILON && t < closest_t { + closest_t = t; + hit_found = true; + + let w0 = 1.0 - u - v; + let w1 = u; + let w2 = v; + + let uv0 = Vec2::from_array(v0.uv); + let uv1 = Vec2::from_array(v1.uv); + let uv2 = Vec2::from_array(v2.uv); + closest_uv = uv0 * w0 + uv1 * w1 + uv2 * w2; + + closest_local_pos = ray_origin + ray_direction * t; + + let n0 = Vec3::from_array(v0.normal); + let n1 = Vec3::from_array(v1.normal); + let n2 = Vec3::from_array(v2.normal); + closest_local_normal = (n0 * w0 + n1 * w1 + n2 * w2).normalize(); + } + } + + if hit_found { + Some(RaycastHit { + uv: closest_uv, + position: closest_local_pos, + normal: closest_local_normal, + distance: closest_t, + }) + } else { + None + } +} \ No newline at end of file diff --git a/app/src/main/rust/src/input.rs b/app/src/main/rust/src/input.rs index 11e1858..551ce1c 100644 --- a/app/src/main/rust/src/input.rs +++ b/app/src/main/rust/src/input.rs @@ -129,7 +129,8 @@ impl InputState { } if let Ok(state) = self.actions.left_thumbstick.state(session, xr::Path::NULL) { - movement[1] = -state.current_state.y; // for some reason down on the thumbstick makes y positive? idk + // my stick drift is actually so bad I can't test with this enabled + // movement[1] = -state.current_state.y; // for some reason down on the thumbstick makes y positive? idk } let mut right_click_state = false; diff --git a/app/src/main/rust/src/jni_state.rs b/app/src/main/rust/src/jni_state.rs index 859611e..ae0c52e 100644 --- a/app/src/main/rust/src/jni_state.rs +++ b/app/src/main/rust/src/jni_state.rs @@ -1,5 +1,5 @@ use { - crate::surface::Surface, + crate::surface::AndroidSurface, ndk_sys::_bindgen_ty_22, jni::{ JavaVM, Env, jni_str, jni_sig, @@ -27,6 +27,7 @@ use { pub static SHOULD_STOP_JNI: AtomicBool = AtomicBool::new(false); pub static PENDING_SKIN_IMAGE: Mutex> = Mutex::new(None); +pub static KEYBOARD_HIDDEN: AtomicBool = AtomicBool::new(true); pub struct SkinData { pub png_bytes: Vec, @@ -42,6 +43,8 @@ pub struct JniContext { method_set_surface: JStaticMethodID, method_process_pointer_event: JStaticMethodID, method_request_ui_render: JStaticMethodID, + method_send_text: JStaticMethodID, + method_delete_character: JStaticMethodID, } impl JniContext { @@ -52,6 +55,8 @@ impl JniContext { let method_system_exit = env.get_static_method_id(&jni_bridge_class, jni_str!("performSystemExit"), RuntimeMethodSignature::from_str("()V").unwrap().method_signature()).unwrap(); let method_process_pointer_event = env.get_static_method_id(&jni_bridge_class, jni_str!("processPointerEvent"), RuntimeMethodSignature::from_str("(IIFF)V").unwrap().method_signature()).unwrap(); let method_request_ui_render = env.get_static_method_id(&jni_bridge_class, jni_str!("requestUiRender"), RuntimeMethodSignature::from_str("()V").unwrap().method_signature()).unwrap(); + let method_send_text = env.get_static_method_id(&jni_bridge_class, jni_str!("sendText"), RuntimeMethodSignature::from_str("(Ljava/lang/String;)V").unwrap().method_signature()).unwrap(); + let method_delete_character = env.get_static_method_id(&jni_bridge_class, jni_str!("deleteCharacter"), RuntimeMethodSignature::from_str("()V").unwrap().method_signature()).unwrap(); JniContext { jvm, @@ -61,7 +66,9 @@ impl JniContext { method_system_exit, method_set_surface, method_process_pointer_event, - method_request_ui_render + method_request_ui_render, + method_send_text, + method_delete_character, } } @@ -87,7 +94,7 @@ impl JniContext { } } - pub fn set_surface(&self, env: &mut Env<'_>, surface: &Surface) { + pub fn set_surface(&self, env: &mut Env<'_>, surface: &AndroidSurface) { let java_surface = jni::sys::jvalue { l: surface.java_surface.as_raw() }; let width = jni::sys::jvalue { i: surface.extent.width as _ }; let height = jni::sys::jvalue { i: surface.extent.height as _ }; @@ -117,6 +124,30 @@ impl JniContext { } } + pub fn send_text(&self, env: &mut Env<'_>, text: &str) { + let text_jstring = env.new_string(text).unwrap(); + let text_jstring = jni::sys::jvalue { l: text_jstring.as_raw() }; + unsafe { + env.call_static_method_unchecked( + &self.jni_bridge_class, + &self.method_send_text, + ReturnType::Primitive(Void), + &[text_jstring] + ).expect("Failed to send text"); + } + } + + pub fn delete_character(&self, env: &mut Env<'_>) { + unsafe { + env.call_static_method_unchecked( + &self.jni_bridge_class, + &self.method_delete_character, + ReturnType::Primitive(Void), + &[], + ).expect("Failed to delete character"); + } + } + pub fn get_internal_files_dir( &self, env: &mut Env<'_> diff --git a/app/src/main/rust/src/keyboard.rs b/app/src/main/rust/src/keyboard.rs new file mode 100644 index 0000000..e882b1d --- /dev/null +++ b/app/src/main/rust/src/keyboard.rs @@ -0,0 +1,384 @@ +use egui::{Color32, Stroke}; +use { + crate::{ + egui::Egui, + render::renderer::{DrawPayload, MSAA_COUNT}, + scene::gltf_model::{GltfAsset, GltfInstance, Vertex}, + stage::Stage, + surface::SurfaceManager, + keyboard::KeyDef::Char, + jni_state::JniContext, + }, + egui::{Frame, Pos2, RawInput, Rect, Button, Event, PointerButton, TextStyle, WidgetText::Text}, + glam::{Mat4, Quat, Vec3, Vec2}, + jni::Env, + log::info, + ndk::asset::AssetManager, + std::sync::Arc, + vk_graph::{ + Graph, + driver::{ + ash::vk::{self, CullModeFlags, Format, PrimitiveTopology}, + buffer::Buffer, + device::Device, + graphics::{BlendInfo, GraphicsPipeline, GraphicsPipelineInfo}, + image::ImageInfo, + shader::{SamplerInfoBuilder, Shader}, + }, + pool::{Pool, lazy::LazyPool}, + }, +}; + +#[derive(Clone, Copy)] +pub enum KeyDef { + Char(&'static str, &'static str), // lowercase, capital + Tab, + CapsLock, + Backspace, + Enter, + Shift, + Space, + Modifier(&'static str), +} + +pub struct Key { + pub def: KeyDef, + pub width: f32, +} +const fn key(def: KeyDef, width: f32) -> Key { + Key { def, width } +} +const ROW_UNITS: f32 = 15.0; + +pub struct KeyboardLayout { // ANSI + pub name: &'static str, + pub row1: &'static [Key], + pub row2: &'static [Key], + pub row3: &'static [Key], + pub row4: &'static [Key], + pub row5: &'static [Key], +} + +impl KeyboardLayout { + pub const QWERTY: KeyboardLayout = KeyboardLayout{ + name: "QWERTY", + row1: &[ + key(Char("`","~"), 1.0), key(Char("1", "!"), 1.0), key(Char("2", "@"), 1.0), key(Char("3","#"), 1.0), key(Char("4", "$"), 1.0), key(Char("5", "%"), 1.0), key(Char("6", "^"), 1.0), key(Char("7", "&"), 1.0), key(Char("8", "*"), 1.0), key(Char("9", "("), 1.0), key(Char("0", ")"), 1.0), key(Char("-", "_"), 1.0), key(Char("=", "+"), 1.0), key(KeyDef::Backspace, 2.0), + ], + row2: &[ + key(KeyDef::Tab, 1.5), key(Char("q", "Q"), 1.0), key(Char("w", "W"), 1.0), key(Char("e", "E"), 1.0), key(Char("r", "R"), 1.0), key(Char("t", "T"), 1.0), key(Char("y", "Y"), 1.0), key(Char("u", "U"), 1.0), key(Char("i", "I"), 1.0), key(Char("o", "O"), 1.0), key(Char("p", "P"), 1.0), key(Char("[", "{"), 1.0), key(Char("]", "}"), 1.0), key(Char("\\", "|"), 1.5), + ], + row3: &[ + key(KeyDef::CapsLock, 1.75), key(Char("a", "A"), 1.0), key(Char("s", "S"), 1.0), key(Char("d", "D"), 1.0), key(Char("f", "F"), 1.0), key(Char("g", "G"), 1.0), key(Char("h", "H"), 1.0), key(Char("j", "J"), 1.0), key(Char("k", "K"), 1.0), key(Char("l", "L"), 1.0), key(Char(";", ":"), 1.0), key(Char("'", "\""), 1.0), key(KeyDef::Enter, 2.25), + ], + row4: &[ + key(KeyDef::Shift, 2.25), key(Char("z", "Z"), 1.0), key(Char("x", "X"), 1.0), key(Char("c", "C"), 1.0), key(Char("v", "V"), 1.0), key(Char("b", "B"), 1.0), key(Char("n", "N"), 1.0), key(Char("m", "M"), 1.0), key(Char(",", "<"), 1.0), key(Char(".", ">"), 1.0), key(Char("/", "?"), 1.0), key(KeyDef::Shift, 2.75), + ], + row5: &[ + key(KeyDef::Modifier(""), 1.25), key(KeyDef::Modifier(""), 1.25), key(KeyDef::Modifier(""), 1.25), key(KeyDef::Space, 6.25), key(KeyDef::Modifier(""), 1.25), key(KeyDef::Modifier(""), 1.25), key(KeyDef::Modifier(""), 1.25), key(KeyDef::Modifier(""), 1.25), + ], + }; +} + +pub struct Keyboard { + pool: LazyPool, + mesh_asset: Arc, + mesh_instance: GltfInstance, + resolution: vk::Extent2D, + jni_context: Arc, + pending_input: RawInput, + + pub surface_manager: SurfaceManager, + pub primitive_transform: Mat4, + pub transform: Mat4, + pub hidden: bool, + pub layout: KeyboardLayout, + pub is_shift: bool, +} + +impl Keyboard { + pub fn new(device: &Device, camera_buffers: &Vec>, asset_manager: &AssetManager, jni_context: Arc, resolution: vk::Extent2D) -> Self { + let gltf_unlit_translucent_shaders = { + let mut asset = asset_manager.open(c"shaders/gltf_unlit_translucent.spv").expect("Failed to load 'gltf_unlit_translucent' shader"); + let spv_bytes = asset.buffer().unwrap(); + + [ + Shader::builder() + .entry_name("vertex_main") + .stage(vk::ShaderStageFlags::VERTEX) + .image_sampler((0, 3), SamplerInfoBuilder::default() + .min_filter(vk::Filter::NEAREST) + .mag_filter(vk::Filter::NEAREST) + .address_mode_u(vk::SamplerAddressMode::REPEAT) + .address_mode_v(vk::SamplerAddressMode::REPEAT) + .address_mode_w(vk::SamplerAddressMode::REPEAT) + .build()) + .vertex_input( + [ + vk::VertexInputBindingDescription { + binding: 0, + stride: size_of::() as u32, + input_rate: vk::VertexInputRate::VERTEX, + } + ], + [ + vk::VertexInputAttributeDescription { + location: 0, + binding: 0, + format: Format::R32G32B32_SFLOAT, + offset: 0, + }, + vk::VertexInputAttributeDescription { + location: 1, + binding: 0, + format: Format::R32G32_SFLOAT, + offset: 12, + }, + vk::VertexInputAttributeDescription { + location: 2, + binding: 0, + format: Format::R32G32B32_SFLOAT, + offset: 20, + }, + vk::VertexInputAttributeDescription { + location: 3, + binding: 0, + format: Format::R32G32B32_SFLOAT, + offset: 32, + }, + vk::VertexInputAttributeDescription { + location: 4, + binding: 0, + format: Format::R32G32B32_SFLOAT, + offset: 44, + }, + vk::VertexInputAttributeDescription { + location: 5, + binding: 0, + format: Format::R32G32B32A32_UINT, + offset: 56, + }, + vk::VertexInputAttributeDescription { + location: 6, + binding: 0, + format: Format::R32G32B32A32_SFLOAT, + offset: 72, + }, + ]) + .spirv(spv_bytes), + Shader::builder() + .entry_name("fragment_main") + .stage(vk::ShaderStageFlags::FRAGMENT) + .spirv(spv_bytes), + ] + }; + let gltf_unlit_translucent_pipeline = Arc::new( + GraphicsPipeline::create(device, GraphicsPipelineInfo::builder() + .topology(PrimitiveTopology::TRIANGLE_LIST) + .blend(BlendInfo::ALPHA) + .samples(MSAA_COUNT) + .cull_mode(CullModeFlags::BACK), gltf_unlit_translucent_shaders.clone()).expect("Failed to create gltf_unlit_translucent pipeline") + ); + let gltf_unlit_translucent_no_cull_pipeline = Arc::new( + GraphicsPipeline::create(device, gltf_unlit_translucent_pipeline.info().into_builder() + .cull_mode(CullModeFlags::NONE), gltf_unlit_translucent_shaders).expect("Failed to create gltf_unlit_translucent_no_cull") + ); + let mesh_asset = Arc::new(GltfAsset::new("Keyboard".to_string(), asset_manager.open(c"meshes/keyboard.glb").unwrap(), device, gltf_unlit_translucent_pipeline, gltf_unlit_translucent_no_cull_pipeline)); + let mesh_instance = GltfInstance::new(mesh_asset.clone(), device, camera_buffers); + let primitive_node = &mesh_instance.nodes[mesh_instance.find_surface_index().expect("Failed to find keyboard surface")]; + let primitive_transform = primitive_node.global_transform.clone(); + + let surface_manager = SurfaceManager::new(asset_manager, device, mesh_asset.clone(), [primitive_node.mesh_index.expect("Keyboard surface does not have a mesh"), 0]); + + let resolution = vk::Extent2D { + width: resolution.width, + height: resolution.height, + }; + info!("Created keyboard of size: {:?}", resolution); + + Self { + pool: LazyPool::new(device), + surface_manager, + mesh_asset, + mesh_instance, + primitive_transform, + resolution, + jni_context, + layout: KeyboardLayout::QWERTY, + transform: Mat4::IDENTITY, + hidden: true, + is_shift: false, + pending_input: RawInput::default(), + } + } + + pub fn adjust_position(&mut self, stage: &Stage, user_height: f32) { + let ideal_user_head_pos = Vec3::new(0.,user_height,0.); + + let local_translation = Vec3::new(0.0, ideal_user_head_pos.y - 0.50, 0.80); + let stage_translation = stage.position + local_translation; + let keyboard_to_user = ideal_user_head_pos - local_translation; + let tilt_angle = keyboard_to_user.y.atan2(-keyboard_to_user.z); + let stage_rotation = stage.rotation * Quat::from_rotation_x(tilt_angle); + + self.transform = Mat4::from_rotation_translation(stage_rotation, stage_translation); + } + + pub fn publish_inputs(&mut self, previous_click_state: bool, current_click_state: bool, uv: Vec2) { + let pos = Pos2 { + x: uv.x * self.resolution.width as f32, + y: uv.y * self.resolution.height as f32, + }; + + self.pending_input.events.push(Event::PointerMoved(pos)); + + if current_click_state && !previous_click_state { + self.pending_input.events.push(Event::PointerButton { + pos, + button: PointerButton::Primary, + pressed: true, + modifiers: Default::default(), + }); + + self.pending_input.events.push(Event::PointerButton { + pos, + button: PointerButton::Primary, + pressed: false, + modifiers: Default::default(), + }); + } + } + + pub fn publish_pointer_leave(&mut self) { + self.pending_input.events.push(Event::PointerGone); + } + + fn keyboard_height(available_width: f32) -> f32 { + let unit = available_width / ROW_UNITS; + let gap = (unit * 0.08).clamp(2.0, 8.0); + let key_height = unit * 0.9; + let row_count = 5.0; + row_count * key_height + (row_count - 1.0) * gap + } + + pub fn record(&mut self, env: &mut Env<'_>, egui: &mut Egui, graph: &mut Graph, draw_payload: &DrawPayload) { + if self.hidden { return } + let target = graph.bind_resource( + self.pool.resource(ImageInfo::image_2d( + self.resolution.width, + self.resolution.height, + Format::R8G8B8A8_UNORM, + vk::ImageUsageFlags::COLOR_ATTACHMENT | vk::ImageUsageFlags::SAMPLED + ).into_builder()).unwrap().with_debug_name("keyboard render target") + ); + + let dims = Pos2{x: self.resolution.width as f32, y: self.resolution.height as f32}; + let mut input = std::mem::take(&mut self.pending_input); + input.screen_rect = Some(Rect { min: Pos2{x: 0., y: 0.}, max: dims } ); + + egui.run(input, target, graph, |ui| { + let frame = Frame::NONE; + egui::Window::new("Keyboard") + .frame(frame) + .resizable(false) + .vscroll(true) + .movable(false) + .title_bar(false) + .default_size([self.resolution.width as f32, self.resolution.height as f32]) + .default_pos([0., 0.]) + .show(ui, |ui| { + let style = ui.style_mut(); + let widgets = &mut style.visuals.widgets; + style.text_styles.get_mut(&TextStyle::Button).expect("missing button text style??").size = 36.0; + widgets.inactive.bg_fill = Color32::from_rgb(0x1A, 0x1A, 0x1A); + widgets.inactive.weak_bg_fill = Color32::from_rgb(0x1A, 0x1A, 0x1A); + widgets.active.bg_fill = Color32::from_rgb(0x4B, 0x67, 0x2E); + widgets.active.weak_bg_fill = Color32::from_rgb(0x4B, 0x67, 0x2E); + widgets.active.bg_stroke = Stroke::new(2.0, Color32::from_rgb(0x7C, 0xB3, 0x42)); + widgets.hovered.bg_fill = Color32::from_rgb(0x1A, 0x1A, 0x1A); + widgets.hovered.weak_bg_fill = Color32::from_rgb(0x1A, 0x1A, 0x1A); + widgets.hovered.bg_stroke = Stroke::new(2.0, Color32::from_rgb(0x7C, 0xB3, 0x42)); + + let rows = [self.layout.row1, self.layout.row2, self.layout.row3, self.layout.row4, self.layout.row5]; + + let unit = ui.available_width() / ROW_UNITS; + let gap = (unit * 0.08).clamp(2.0, 8.0); + let key_height = unit * 0.9; + let row_count = rows.len() as f32; + let content_height = row_count * key_height + (row_count - 1.0) * gap; + + let top_pad = ((ui.available_height() - content_height) / 2.0).max(0.0); + ui.add_space(top_pad); + ui.spacing_mut().item_spacing = egui::vec2(gap, gap); + + ui.vertical(|ui| { + for row in rows { + ui.horizontal(|ui| { + for k in row { + let width = unit * k.width - gap; + match k.def { + Char(lower, upper) => { + let label_text = if self.is_shift { upper } else { lower }; + let label = Text(label_text.to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label)).clicked() { + self.jni_context.send_text(env, label_text); + } + } + KeyDef::Tab => { + let label = Text("Tab".to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label)).clicked() { + self.jni_context.send_text(env, "\t"); + } + } + KeyDef::CapsLock => { + let label = Text("Caps".to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label).selected(self.is_shift)).clicked() { + self.is_shift = !self.is_shift; + } + } + KeyDef::Backspace => { + let label = Text("⌫".to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label)).clicked() { + self.jni_context.delete_character(env); + } + } + KeyDef::Enter => { + let label = Text("Enter".to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label)).clicked() { + self.jni_context.send_text(env, "\n"); + } + } + KeyDef::Shift => { + let label = Text("⇧ Shift".to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label).selected(self.is_shift)).clicked() { + self.is_shift = !self.is_shift; + } + } + KeyDef::Space => { + let label = Text("Space".to_string()).text_style(TextStyle::Button); + if ui.add_sized([width, key_height], Button::new(label)).clicked() { + self.jni_context.send_text(env, " "); + } + } + KeyDef::Modifier(label) => { + if label.is_empty() { + ui.add_space(width) + } else { + let label = Text(label.to_string()).text_style(TextStyle::Button); + ui.add_enabled_ui(false, |ui| { + ui.add_sized([width, key_height], Button::new(label)); + }); + } + } + } + } + }); + } + }); + }); + }); + + self.mesh_instance.record_with_transform(graph, draw_payload, &self.transform); + self.surface_manager.record_with_transform_bound(graph, target.into(), draw_payload, &(self.transform * self.primitive_transform)); + } +} \ No newline at end of file diff --git a/app/src/main/rust/src/lib.rs b/app/src/main/rust/src/lib.rs index f3997fd..fac896a 100644 --- a/app/src/main/rust/src/lib.rs +++ b/app/src/main/rust/src/lib.rs @@ -17,6 +17,8 @@ mod stage; mod xr_util; mod surface; mod egui; +mod keyboard; +mod geometry; #[jni_mangle("com.qcxr.questcraft.JniBridge")] pub fn start<'local>( @@ -40,10 +42,16 @@ pub fn start<'local>( .or_else(|| panic_info.payload().downcast_ref::().map(|s| s.as_str())) .unwrap_or("Unknown panic payload"); + let location_info = if let Some(location) = panic_info.location() { + format!("{}:{}:{}", location.file(), location.line(), location.column()) + } else { + "unknown location".to_string() + }; + let thread_name = std::thread::current().name().unwrap_or("main").to_string(); - let error = format!("Exception in thread \"{}\": {}\n", thread_name, message); + let error = format!("Exception in thread \"{}\" at {}: {}\n", thread_name, location_info, message); - log::error!("!! PANIC !!\n{}", error); + log::error!("!! PANIC !! {}", error); })); unowned_env.with_env(|env| -> jni::errors::Result<_> { @@ -72,11 +80,25 @@ pub fn start<'local>( #[jni_mangle("com.qcxr.questcraft.JniBridge")] pub fn stop( - mut _unowned_env: EnvUnowned, + _unowned_env: EnvUnowned, ) { jni_state::SHOULD_STOP_JNI.store(true, Ordering::Relaxed); } +#[jni_mangle("com.qcxr.questcraft.JniBridge")] +pub fn show_keyboard( + _unowned_env: EnvUnowned, +) { + jni_state::KEYBOARD_HIDDEN.store(false, Ordering::Relaxed); +} + +#[jni_mangle("com.qcxr.questcraft.JniBridge")] +pub fn hide_keyboard( + _unowned_env: EnvUnowned, +) { + jni_state::KEYBOARD_HIDDEN.store(true, Ordering::Relaxed); +} + #[jni_mangle("com.qcxr.questcraft.JniBridge")] pub fn set_skin_image<'local>( mut unowned_env: EnvUnowned<'local>, diff --git a/app/src/main/rust/src/render/renderer.rs b/app/src/main/rust/src/render/renderer.rs index 4e26c88..b89e2d8 100644 --- a/app/src/main/rust/src/render/renderer.rs +++ b/app/src/main/rust/src/render/renderer.rs @@ -1,4 +1,8 @@ use std::path::Path; +use std::sync::Arc; +use log::warn; +use vk_graph::driver::buffer::Buffer; +use vk_graph::node::{AnyBufferNode, AnyImageNode}; use { crate::{ app::XrContext, @@ -16,8 +20,7 @@ use { vk_graph::{ cmd::ClearColorValue, driver::{ - ash::vk::{self, BufferUsageFlags, DeviceSize}, - buffer::BufferInfo, + ash::vk::{self}, device::Device, fence::Fence, image::{ImageInfo, SampleCount}, @@ -38,9 +41,11 @@ pub enum RenderingError { } pub struct Renderer<'a> { - resolution: vk::Extent2D, + pub resolution: vk::Extent2D, - pool: LazyPool, + pub pool: LazyPool, + pub frame_in_flight: usize, + max_frames_in_flight: usize, swapchain_queues: Box<[Option]>, swapchain_rect: xr::Rect2Di, fixture_path: &'a Path, @@ -53,7 +58,7 @@ pub enum AcquireError { } #[derive(Debug, PartialEq, Eq, Copy, Clone)] pub enum PollError { - Exiting, LossPending, + Exiting, LossPending, Recenter } #[derive(Debug, PartialEq, Eq, Copy, Clone)] pub enum WaitError { @@ -73,15 +78,15 @@ pub struct ActiveFrame { } pub struct FramePayload<'a> { - pub view_matrices: [Mat4; 2], - pub projection_matrices: [Mat4; 2], + pub camera_buffer: &'a Arc, pub xr_views: &'a [openxr::View], } pub struct DrawPayload<'a> { - pub camera_ubo: &'a vk_graph::node::AnyBufferNode, - pub color_target: &'a vk_graph::node::AnyImageNode, - pub depth_target: &'a vk_graph::node::AnyImageNode, + pub frame_in_flight: usize, + pub camera_ubo: &'a AnyBufferNode, + pub color_target: &'a AnyImageNode, + pub depth_target: &'a AnyImageNode, } pub const VIEW_MASK: u32 = !(!0 << 2); @@ -93,7 +98,9 @@ impl<'a> Renderer<'a> { return Err(Sleeping); } - let frame_state = xr_context.session.wait_frame().map_err(|err| if err == WaitError::Sleeping { Sleeping } else { RenderingError::DriverError })?; + let frame_state = xr_context.session.wait_frame().map_err(|err| { + if err == WaitError::Sleeping { Sleeping } else { RenderingError::DriverError } + })?; let (_, swapchain_image_index) = xr_context.swapchain.acquire_image().unwrap(); Ok(ActiveFrame{ @@ -123,22 +130,13 @@ impl<'a> Renderer<'a> { ).into_builder().sample_count(MSAA_COUNT)).unwrap().with_debug_name("main depth image") ); - let camera_data = XrCameraUBO { - view_matrices: payload.view_matrices, - projection_matrices: payload.projection_matrices, - }; - let mut ubo_buffer = self.pool.resource(BufferInfo::builder() - .host_writable(true) - .size(size_of::() as DeviceSize) - .usage(BufferUsageFlags::UNIFORM_BUFFER) - ).map_err(|_| RenderingError::DriverError)?; - ubo_buffer.copy_from_slice(0, bytemuck::bytes_of(&camera_data)); - let camera_ubo_node = graph.bind_resource(ubo_buffer); + let camera_ubo_node = graph.bind_resource(payload.camera_buffer); graph.clear_color_image(swapchain_image, ClearColorValue::WHITE_ALPHA_ONE); graph.clear_depth_stencil_image(depth_target, 1.0, 0); let draw_payload = DrawPayload { + frame_in_flight: self.frame_in_flight, camera_ubo: &camera_ubo_node.into(), color_target: &swapchain_image.into(), depth_target: &depth_target.into(), @@ -165,13 +163,14 @@ impl<'a> Renderer<'a> { active_frame.predicted_display_time, EnvironmentBlendMode::OPAQUE, &[&xr::CompositionLayerProjection::new() - .space(&xr_context.stage) + .space(&xr_context.stage_space) .views(&[ xr::CompositionLayerProjectionView::new().pose(payload.xr_views[0].pose).fov(payload.xr_views[0].fov).sub_image(xr::SwapchainSubImage::new().swapchain(&xr_context.swapchain).image_array_index(0).image_rect(self.swapchain_rect)), xr::CompositionLayerProjectionView::new().pose(payload.xr_views[1].pose).fov(payload.xr_views[1].fov).sub_image(xr::SwapchainSubImage::new().swapchain(&xr_context.swapchain).image_array_index(1).image_rect(self.swapchain_rect)), ]) ] ).map_err(|_| FailedToEndStream)?; + self.frame_in_flight = (self.frame_in_flight + 1) % self.max_frames_in_flight; #[cfg(feature = "profiled")] profiling::finish_frame!(); @@ -197,6 +196,8 @@ impl<'a> Renderer<'a> { Renderer { resolution, pool, + frame_in_flight: 0, + max_frames_in_flight: swapchain_image_count, swapchain_queues, swapchain_rect, fixture_path, diff --git a/app/src/main/rust/src/scene/gltf_model.rs b/app/src/main/rust/src/scene/gltf_model.rs index 12e3d6b..44a6b64 100644 --- a/app/src/main/rust/src/scene/gltf_model.rs +++ b/app/src/main/rust/src/scene/gltf_model.rs @@ -1,3 +1,4 @@ +use vk_graph::driver::descriptor_set::{DescriptorSet, DescriptorSetBinding, DescriptorSetInfo, DescriptorSetUpdateInfo}; use { std::{collections::HashMap, sync::Arc}, bytemuck::{bytes_of, Pod, Zeroable}, @@ -725,8 +726,8 @@ impl GltfAsset { if let Some(extras) = mesh.extras() { let json_str = extras.get(); if let Ok(ui_props) = serde_json::from_str::(json_str) { - if ui_props.is_ui_surface.is_some() { - info!("Found UI surface mesh: {:?}", node.name().or(node.mesh().and_then(|m| m.name()))); + if ui_props.is_ui_surface.is_some() || ui_props.is_grab_bar.is_some() { + info!("Found special mesh: {:?}", node.name().or(node.mesh().and_then(|m| m.name()))); specials.push(nodes.len()); node_extras.insert(nodes.len(), ui_props); } @@ -815,13 +816,15 @@ pub struct GltfInstance { pub asset: Arc, pub nodes: Vec, + descriptor_sets: Vec, + instance_data: Vec, instance_buffer: Arc, joint_matrix_buffer: Arc, } impl GltfInstance { - pub fn new(asset: Arc, device: &Device) -> Self { + pub fn new(asset: Arc, device: &Device, camera_buffers: &Vec>) -> Self { let nodes = asset.bind_pose_nodes.clone(); let instance_data = asset.base_instance_data.clone(); @@ -838,14 +841,81 @@ impl GltfInstance { bytemuck::cast_slice(&joint_matrix_seed), ).expect("Failed to create joint matrix buffer")); + let mut descriptor_sets = Vec::new(); + + for camera_buffer in camera_buffers { + let mut descriptors = Vec::new(); + descriptors.push(DescriptorSetUpdateInfo::buffer(0, camera_buffer)); + descriptors.push(DescriptorSetUpdateInfo::buffer(1, &instance_buffer)); + descriptors.push(DescriptorSetUpdateInfo::buffer(2, &joint_matrix_buffer)); + + for (idx, texture) in asset.textures.iter().enumerate() { + descriptors.push(DescriptorSetUpdateInfo::image(DescriptorSetBinding { binding: 3, array_element: idx as u32 }, texture)); + } + + descriptor_sets.push(DescriptorSet::alloc_and_update( + &*asset.culled_pipeline, + DescriptorSetInfo::builder().set(0).build(), + descriptors + ).expect("Failed to allocate and update descriptor set 0")); + } + GltfInstance { asset, nodes, + descriptor_sets, instance_data, instance_buffer, joint_matrix_buffer, } } + + pub fn override_textures(&mut self, texture: &Arc, camera_buffers: &Vec>) { + let mut descriptor_sets = Vec::new(); + + for camera_buffer in camera_buffers { + let mut descriptors = Vec::new(); + descriptors.push(DescriptorSetUpdateInfo::buffer(0, camera_buffer)); + descriptors.push(DescriptorSetUpdateInfo::buffer(1, &self.instance_buffer)); + descriptors.push(DescriptorSetUpdateInfo::buffer(2, &self.joint_matrix_buffer)); + + for (idx, _) in self.asset.textures.iter().enumerate() { + descriptors.push(DescriptorSetUpdateInfo::image(DescriptorSetBinding { binding: 3, array_element: idx as u32 }, texture)); + } + + descriptor_sets.push(DescriptorSet::alloc_and_update( + &*self.asset.culled_pipeline, + DescriptorSetInfo::builder().set(0).build(), + descriptors + ).expect("Failed to allocate and update descriptor set 0")); + } + + self.descriptor_sets = descriptor_sets; + } + + // note: this isn't really normally necessary to use, but it's here in case we'd ever want to reset the textures for whatever reason. + pub fn recreate_descriptor_sets(&mut self, camera_buffers: &Vec>) { + let mut descriptor_sets = Vec::new(); + + for camera_buffer in camera_buffers { + let mut descriptors = Vec::new(); + descriptors.push(DescriptorSetUpdateInfo::buffer(0, camera_buffer)); + descriptors.push(DescriptorSetUpdateInfo::buffer(1, &self.instance_buffer)); + descriptors.push(DescriptorSetUpdateInfo::buffer(2, &self.joint_matrix_buffer)); + + for (idx, texture) in self.asset.textures.iter().enumerate() { + descriptors.push(DescriptorSetUpdateInfo::image(DescriptorSetBinding { binding: 3, array_element: idx as u32 }, texture)); + } + + descriptor_sets.push(DescriptorSet::alloc_and_update( + &*self.asset.culled_pipeline, + DescriptorSetInfo::builder().set(0).build(), + descriptors + ).expect("Failed to allocate and update descriptor set 0")); + } + + self.descriptor_sets = descriptor_sets; + } pub fn create_animation_player(&self, clip_name: &str, looping: bool) -> Option { self.asset.find_animation(clip_name).map(|idx| AnimationPlayer::new(idx, looping)) @@ -918,10 +988,11 @@ impl GltfInstance { .begin_cmd() .debug_name(format!("Scene {} (culled)", asset.identifier)) .bind_pipeline(&*asset.culled_pipeline) + .bind_descriptor_set(&self.descriptor_sets[draw_payload.frame_in_flight]) .multiview(crate::render::renderer::VIEW_MASK, 0) - .shader_resource_access(0, *draw_payload.camera_ubo, AccessType::VertexShaderReadUniformBuffer) - .shader_resource_access(1, instance_node, AccessType::VertexShaderReadOther) - .shader_resource_access(2, joint_node, AccessType::VertexShaderReadOther) + .resource_access(*draw_payload.camera_ubo, AccessType::VertexShaderReadUniformBuffer) + .resource_access(instance_node, AccessType::VertexShaderReadOther) + .resource_access(joint_node, AccessType::VertexShaderReadOther) .color_attachment_image(0, *draw_payload.color_target, LoadOp::Load, StoreOp::Store) .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .depth_stencil_attachment_image(*draw_payload.depth_target, LoadOp::Load, StoreOp::Store) @@ -929,10 +1000,9 @@ impl GltfInstance { .resource_access(v_node, AccessType::VertexBuffer) .resource_access(indirect_node, AccessType::IndirectBuffer); - for (idx, texture) in asset.textures.iter().enumerate() { + for texture in &asset.textures { let image_node = cmd_builder.bind_resource(texture); - cmd_builder.set_shader_resource_access( - (3, [idx as u32]), + cmd_builder.set_resource_access( image_node, AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer, ); @@ -952,10 +1022,11 @@ impl GltfInstance { .begin_cmd() .debug_name(format!("Scene {} (no-cull)", asset.identifier)) .bind_pipeline(&*asset.no_cull_pipeline) + .bind_descriptor_set(&self.descriptor_sets[draw_payload.frame_in_flight]) .multiview(crate::render::renderer::VIEW_MASK, 0) - .shader_resource_access(0, *draw_payload.camera_ubo, AccessType::VertexShaderReadUniformBuffer) - .shader_resource_access(1, instance_node, AccessType::VertexShaderReadOther) - .shader_resource_access(2, joint_node, AccessType::VertexShaderReadOther) + .resource_access(*draw_payload.camera_ubo, AccessType::VertexShaderReadUniformBuffer) + .resource_access(instance_node, AccessType::VertexShaderReadOther) + .resource_access(joint_node, AccessType::VertexShaderReadOther) .color_attachment_image(0, *draw_payload.color_target, LoadOp::Load, StoreOp::Store) .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) .depth_stencil_attachment_image(*draw_payload.depth_target, LoadOp::Load, StoreOp::Store) @@ -963,94 +1034,9 @@ impl GltfInstance { .resource_access(v_node, AccessType::VertexBuffer) .resource_access(indirect_node, AccessType::IndirectBuffer); - for (idx, texture) in asset.textures.iter().enumerate() { + for texture in &asset.textures { let image_node = cmd_builder.bind_resource(texture); - cmd_builder.set_shader_resource_access( - (3, [idx as u32]), - image_node, - AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer, - ); - } - - let draw_count = asset.no_cull_draw_count; - cmd_builder.record_cmd(move |cmd| { - cmd.bind_index_buffer(i_node, 0, IndexType::UINT32) - .bind_vertex_buffer(0, v_node, 0) - .push_constants(0, bytes_of(&scene_transform)) - .draw_indexed_indirect(indirect_node, no_cull_offset, draw_count, stride as u32); - }).end_cmd(); - } - } - - pub fn record_with_transform_override_texture(&self, graph: &mut Graph, draw_payload: &DrawPayload, transform: &Mat4, texture: Arc) { - #[cfg(feature = "profiled")] - profiling::function_scope!(); - - let asset = &self.asset; - let v_node = graph.bind_resource(asset.vertex_buffer.clone()); - let i_node = graph.bind_resource(asset.index_buffer.clone()); - let instance_node = graph.bind_resource(self.instance_buffer.clone()); - let indirect_node = graph.bind_resource(asset.indirect_buffer.clone()); - let joint_node = graph.bind_resource(self.joint_matrix_buffer.clone()); - - let scene_transform = *transform; - let stride = size_of::() as vk::DeviceSize; - let no_cull_offset = stride * asset.culled_draw_count as vk::DeviceSize; - - if asset.culled_draw_count > 0 { - let mut cmd_builder = graph - .begin_cmd() - .debug_name(format!("Scene {} (culled)", asset.identifier)) - .bind_pipeline(&*asset.culled_pipeline) - .multiview(crate::render::renderer::VIEW_MASK, 0) - .shader_resource_access(0, *draw_payload.camera_ubo, AccessType::VertexShaderReadUniformBuffer) - .shader_resource_access(1, instance_node, AccessType::VertexShaderReadOther) - .shader_resource_access(2, joint_node, AccessType::VertexShaderReadOther) - .color_attachment_image(0, *draw_payload.color_target, LoadOp::Load, StoreOp::Store) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) - .depth_stencil_attachment_image(*draw_payload.depth_target, LoadOp::Load, StoreOp::Store) - .resource_access(i_node, AccessType::IndexBuffer) - .resource_access(v_node, AccessType::VertexBuffer) - .resource_access(indirect_node, AccessType::IndirectBuffer); - - let image_node = cmd_builder.bind_resource(&texture); - for (idx, _) in asset.textures.iter().enumerate() { - cmd_builder.set_shader_resource_access( - (3, [idx as u32]), - image_node, - AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer, - ); - } - - let draw_count = asset.culled_draw_count; - cmd_builder.record_cmd(move |cmd| { - cmd.bind_index_buffer(i_node, 0, IndexType::UINT32) - .bind_vertex_buffer(0, v_node, 0) - .push_constants(0, bytes_of(&scene_transform)) - .draw_indexed_indirect(indirect_node, 0, draw_count, stride as u32); - }).end_cmd(); - } - - if asset.no_cull_draw_count > 0 { - let mut cmd_builder = graph - .begin_cmd() - .debug_name(format!("Scene {} (no-cull)", asset.identifier)) - .bind_pipeline(&*asset.no_cull_pipeline) - .multiview(crate::render::renderer::VIEW_MASK, 0) - .shader_resource_access(0, *draw_payload.camera_ubo, AccessType::VertexShaderReadUniformBuffer) - .shader_resource_access(1, instance_node, AccessType::VertexShaderReadOther) - .shader_resource_access(2, joint_node, AccessType::VertexShaderReadOther) - .color_attachment_image(0, *draw_payload.color_target, LoadOp::Load, StoreOp::Store) - .depth_stencil(DepthStencilInfo::DEPTH_WRITE_LESS) - .depth_stencil_attachment_image(*draw_payload.depth_target, LoadOp::Load, StoreOp::Store) - .resource_access(i_node, AccessType::IndexBuffer) - .resource_access(v_node, AccessType::VertexBuffer) - .resource_access(indirect_node, AccessType::IndirectBuffer); - - let image_node = cmd_builder.bind_resource(&texture); - for (idx, _) in asset.textures.iter().enumerate() { - cmd_builder.set_shader_resource_access( - (3, [idx as u32]), + cmd_builder.set_resource_access( image_node, AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer, ); @@ -1088,6 +1074,17 @@ impl GltfInstance { None }) } + + pub fn find_grab_bar_index(&self) -> Option { + self.asset.specials.iter().find_map(|&idx| { + if let Some(extras) = self.asset.node_extras.get(&idx) { + if extras.is_grab_bar == Some(1) { + return Some(idx) + } + } + None + }) + } } #[derive(Clone, Copy, Pod, Zeroable)] @@ -1176,4 +1173,6 @@ pub struct Extras { pub is_ui_surface: Option, #[serde(rename = "is_spawnpoint")] pub is_spawnpoint: Option, + #[serde(rename = "is_grab_bar")] + pub is_grab_bar: Option, } \ No newline at end of file diff --git a/app/src/main/rust/src/scene/scene.rs b/app/src/main/rust/src/scene/scene.rs index 2bf2a66..f4da72d 100644 --- a/app/src/main/rust/src/scene/scene.rs +++ b/app/src/main/rust/src/scene/scene.rs @@ -1,3 +1,4 @@ +use vk_graph::driver::buffer::Buffer; use { crate::{ render::renderer::{self, DrawPayload}, @@ -44,15 +45,15 @@ pub struct Assets { pub animated_asset: Arc, pub animated_instance: GltfInstance, - // these _asset fields don't really need to be here because GltfInstance holds a reference, but I don't really want to delete them in case we want to add animations to them (which would require an accessible reference to the asset) + // these _asset fields don't really need to be here because GltfInstance holds a reference. However, I don't really want to delete them in case we want to add animations to them (which would require an accessible reference to the asset) left_controller_scene_asset: Arc, - left_controller_scene_instance: GltfInstance, + pub left_controller_scene_instance: GltfInstance, right_controller_scene_asset: Arc, - right_controller_scene_instance: GltfInstance, + pub right_controller_scene_instance: GltfInstance, slim_left_controller_scene_asset: Arc, - slim_left_controller_scene_instance: GltfInstance, + pub slim_left_controller_scene_instance: GltfInstance, slim_right_controller_scene_asset: Arc, - slim_right_controller_scene_instance: GltfInstance, + pub slim_right_controller_scene_instance: GltfInstance, ray_scene_asset: Arc, ray_scene_instance: GltfInstance, @@ -68,7 +69,7 @@ pub struct Scene { } impl Scene { - pub fn load(device: &Device, asset_manager: &AssetManager) -> Self { + pub fn load(device: &Device, camera_buffers: &Vec>, asset_manager: &AssetManager) -> Self { let gltf_unlit_shaders = { let mut asset = asset_manager.open(c"shaders/gltf_unlit.spv").expect("Failed to load 'gltf_unlit' shader"); let spv_bytes = asset.buffer().unwrap(); @@ -244,47 +245,47 @@ impl Scene { let asset = asset_manager.open(c"meshes/scene.glb").expect("Failed to load 'scene.glb'"); Arc::new(GltfAsset::new("Test".to_string(), asset, device, gltf_unlit_pipeline.clone(), gltf_unlit_no_cull_pipeline.clone())) }; - let scene_instance = GltfInstance::new(scene_asset.clone(), device); + let scene_instance = GltfInstance::new(scene_asset.clone(), device, camera_buffers); let animated_asset = { let asset = asset_manager.open(c"meshes/animated.glb").expect("Failed to load 'animated.glb'"); Arc::new(GltfAsset::new("Animated".to_string(), asset, device, gltf_unlit_translucent_pipeline.clone(), gltf_unlit_translucent_no_cull_pipeline.clone())) }; - let animated_instance = GltfInstance::new(animated_asset.clone(), device); + let animated_instance = GltfInstance::new(animated_asset.clone(), device, camera_buffers); let left_controller_scene_asset = { let asset = asset_manager.open(c"meshes/left_controller.glb").expect("Failed to load 'left_controller.glb'"); Arc::new(GltfAsset::new("Controller".to_string(), asset, device, gltf_unlit_pipeline.clone(), gltf_unlit_no_cull_pipeline.clone())) }; - let left_controller_scene_instance = GltfInstance::new(left_controller_scene_asset.clone(), device); + let left_controller_scene_instance = GltfInstance::new(left_controller_scene_asset.clone(), device, camera_buffers); let right_controller_scene_asset = { let asset = asset_manager.open(c"meshes/right_controller.glb").expect("Failed to load 'right_controller.glb'"); Arc::new(GltfAsset::new("Controller".to_string(), asset, device, gltf_unlit_pipeline.clone(), gltf_unlit_no_cull_pipeline.clone())) }; - let right_controller_scene_instance = GltfInstance::new(right_controller_scene_asset.clone(), device); + let right_controller_scene_instance = GltfInstance::new(right_controller_scene_asset.clone(), device, camera_buffers); let slim_left_controller_scene_asset = { let asset = asset_manager.open(c"meshes/slim_left_controller.glb").expect("Failed to load 'slim_left_controller.glb'"); Arc::new(GltfAsset::new("Controller".to_string(), asset, device, gltf_unlit_pipeline.clone(), gltf_unlit_no_cull_pipeline.clone())) }; - let slim_left_controller_scene_instance = GltfInstance::new(slim_left_controller_scene_asset.clone(), device); + let slim_left_controller_scene_instance = GltfInstance::new(slim_left_controller_scene_asset.clone(), device, camera_buffers); let slim_right_controller_scene_asset = { let asset = asset_manager.open(c"meshes/slim_right_controller.glb").expect("Failed to load 'slim_right_controller.glb'"); Arc::new(GltfAsset::new("Controller".to_string(), asset, device, gltf_unlit_pipeline.clone(), gltf_unlit_no_cull_pipeline.clone())) }; - let slim_right_controller_scene_instance = GltfInstance::new(slim_right_controller_scene_asset.clone(), device); + let slim_right_controller_scene_instance = GltfInstance::new(slim_right_controller_scene_asset.clone(), device, camera_buffers); let ray_scene_asset = { let asset = asset_manager.open(c"meshes/ray.glb").expect("Failed to load 'ray.glb'"); Arc::new(GltfAsset::new("Ray".to_string(), asset, device, gltf_unlit_translucent_pipeline.clone(), gltf_unlit_translucent_no_cull_pipeline.clone())) }; - let ray_scene_instance = GltfInstance::new(ray_scene_asset.clone(), device); + let ray_scene_instance = GltfInstance::new(ray_scene_asset.clone(), device, camera_buffers); let pointer_scene_asset = { let asset = asset_manager.open(c"meshes/pointer.glb").expect("Failed to load 'pointer.glb'"); Arc::new(GltfAsset::new("Pointer".to_string(), asset, device, gltf_unlit_translucent_pipeline.clone(), gltf_unlit_translucent_no_cull_pipeline.clone())) }; - let pointer_scene_instance = GltfInstance::new(pointer_scene_asset.clone(), device); + let pointer_scene_instance = GltfInstance::new(pointer_scene_asset.clone(), device, camera_buffers); let spawn_matrix = scene_instance.find_spawnpoint_transform(); if let Some(spawn_matrix) = &spawn_matrix { @@ -344,11 +345,8 @@ impl Scene { } }; - if let Some(ref skin) = *skin { - scene.record_with_transform_override_texture(graph, draw_payload, controller_matrix, skin.texture.clone()); - } else { - scene.record_with_transform(graph, draw_payload, controller_matrix); - } + + scene.record_with_transform(graph, draw_payload, controller_matrix); if let Some(ray_transform) = ray_transform { self.assets.ray_scene_instance.record_with_transform(graph, draw_payload, &ray_transform); } diff --git a/app/src/main/rust/src/surface.rs b/app/src/main/rust/src/surface.rs index b11a1e9..302721c 100644 --- a/app/src/main/rust/src/surface.rs +++ b/app/src/main/rust/src/surface.rs @@ -1,25 +1,27 @@ use { crate::{ + geometry::{self, RaycastHit}, render::renderer::{self, DrawPayload}, - scene::gltf_model::{self, GltfPrimitive, GltfAsset} + scene::gltf_model::{self, GltfAsset} }, bytemuck::{Pod, Zeroable, bytes_of}, - glam::{Mat4, Vec2, Vec3}, + glam::{Mat4, Vec3}, jni::{Env, objects::JObject, refs::Global}, ndk::asset::AssetManager, - ndk_sys::{AHardwareBuffer, AHardwareBuffer_acquire, AHardwareBuffer_release, AImageReader, AImageReader_acquireLatestImage, AImageReader_delete, AImageReader_getWindow, AImageReader_newWithUsage, AImage_delete, AImage_getHardwareBuffer, ANativeWindow_toSurface, media_status_t}, + ndk_sys::{AHardwareBuffer, AHardwareBuffer_UsageFlags, AHardwareBuffer_acquire, AHardwareBuffer_release, AIMAGE_FORMATS, AImageReader, AImageReader_acquireLatestImage, AImageReader_delete, AImageReader_getWindow, AImageReader_newWithUsage, AImage_delete, AImage_getHardwareBuffer, ANativeWindow_toSurface, media_status_t}, std::sync::Arc, vk_graph::{ + Graph, LoadOp, StoreOp, driver::{ ash::vk::{self, Format, ImageViewType, IndexType, PrimitiveTopology}, device::Device, - graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo}, + graphics::{DepthStencilInfo, GraphicsPipeline, GraphicsPipelineInfo, BlendInfo}, image::{Image, ImageInfo, ImageViewInfoBuilder, SampleCount}, shader::{SamplerInfoBuilder, Shader}, - sync::AccessType + sync::AccessType, }, - Graph, LoadOp, StoreOp - } + node::AnyImageNode, + }, }; pub struct SurfaceTexture { @@ -39,7 +41,7 @@ impl Drop for SurfaceTexture { } } -pub struct Surface { +pub struct AndroidSurface { pub extent: vk::Extent3D, pub image_reader: *mut AImageReader, pub java_surface: Global>, @@ -55,16 +57,15 @@ fn find_memory_type_index(device: &Device, type_bits: u32, properties: vk::Memor panic!("Failed to find suitable memory type for AHardwareBuffer, properties: {:?}", mem_properties); } -impl Surface { +impl AndroidSurface { pub fn new(env: &mut Env<'_>, width: u32, height: u32) -> Self { - let image_reader = unsafe { let mut image_reader = std::ptr::null_mut(); let status = AImageReader_newWithUsage( width as i32, height as i32, - 0x00000001, // this is format RBGA_8888, see https://developer.android.com/reference/android/graphics/ImageFormat for constant values - (1u64 << 8) | (1u64 << 9), // this is usage GPU_SAMPLED_IMAGE, see https://developer.android.com/ndk/reference/group/a-hardware-buffer for constant values + AIMAGE_FORMATS::AIMAGE_FORMAT_RGBA_8888.0 as i32, + AHardwareBuffer_UsageFlags::AHARDWAREBUFFER_USAGE_GPU_SAMPLED_IMAGE.0 as u64 | AHardwareBuffer_UsageFlags::AHARDWAREBUFFER_USAGE_GPU_FRAMEBUFFER.0 as u64, 4, &mut image_reader ); @@ -228,7 +229,7 @@ impl Surface { } } -impl Drop for Surface { +impl Drop for AndroidSurface { fn drop(&mut self) { unsafe { if !self.image_reader.is_null() { @@ -238,26 +239,20 @@ impl Drop for Surface { } } -pub struct SurfaceManager<'a> { +pub struct SurfaceManager { pub pipeline: Arc, - pub scene: &'a GltfAsset, - pub primitive: &'a GltfPrimitive, + pub scene: Arc, + pub primitive_index: [usize; 2], } #[derive(Clone, Copy, Pod, Zeroable)] #[repr(C)] pub struct SurfacePushConstants { - model_transform: Mat4, -} - -pub struct RaycastHit { - pub uv: Vec2, - pub world_position: Vec3, - pub world_normal: Vec3, + pub(crate) model_transform: Mat4, } -impl<'a> SurfaceManager<'a> { - pub fn new(asset_manager: &AssetManager, device: &Device, scene: &'a GltfAsset, primitive: &'a GltfPrimitive) -> Self { +impl SurfaceManager { + pub fn new(asset_manager: &AssetManager, device: &Device, scene: Arc, primitive_index: [usize; 2]) -> Self { let pipeline = Arc::new({ let mut asset = asset_manager.open(c"shaders/gltf_surface.spv").expect("Failed to load 'gltf_surface' shader"); let spv_bytes = asset.buffer().unwrap(); @@ -265,6 +260,7 @@ impl<'a> SurfaceManager<'a> { GraphicsPipeline::create(device, GraphicsPipelineInfo::builder() .topology(PrimitiveTopology::TRIANGLE_LIST) .cull_mode(vk::CullModeFlags::NONE) + .blend(BlendInfo::ALPHA) .samples(renderer::MSAA_COUNT) .min_sample_shading(0.5), [ Shader::builder() @@ -309,15 +305,16 @@ impl<'a> SurfaceManager<'a> { Self { scene, - primitive, + primitive_index, pipeline, } } - // this uses the Moller-Trumbore intersection algorithm + // this transforms the raycast to world space from local space after it runs pub fn raycast_uv(&self, controller_world_matrix: Mat4, surface_world_matrix: Mat4, ray_direction: Vec3) -> Option { - let vertices = self.primitive.cpu_vertex_buffer.as_ref()?; - let indices = self.primitive.cpu_index_buffer.as_ref()?; + let prim = &self.scene.meshes[self.primitive_index[0]].primitives[self.primitive_index[1]]; + let vertices = prim.cpu_vertex_buffer.as_ref()?; + let indices = prim.cpu_index_buffer.as_ref()?; let ray_origin_world = controller_world_matrix.transform_point3(Vec3::ZERO); let ray_direction_world = controller_world_matrix.transform_vector3(ray_direction).normalize(); @@ -326,83 +323,27 @@ impl<'a> SurfaceManager<'a> { let ray_origin = world_to_mesh_local.transform_point3(ray_origin_world); let ray_direction = world_to_mesh_local.transform_vector3(ray_direction_world).normalize(); - let mut closest_t = f32::INFINITY; - - let mut closest_local_pos = Vec3::ZERO; - let mut closest_local_normal = Vec3::ZERO; - let mut closest_uv = Vec2::ZERO; - let mut hit_found = false; - - for chunk in indices.chunks_exact(3) { - let v0 = &vertices[chunk[0] as usize]; - let v1 = &vertices[chunk[1] as usize]; - let v2 = &vertices[chunk[2] as usize]; - - let p0 = Vec3::from_array(v0.position); - let p1 = Vec3::from_array(v1.position); - let p2 = Vec3::from_array(v2.position); - - let edge1 = p1 - p0; - let edge2 = p2 - p0; - let h = ray_direction.cross(edge2); - let a = edge1.dot(h); - - if a.abs() < f32::EPSILON { continue; } - - let f = 1.0 / a; - let s = ray_origin - p0; - let u = f * s.dot(h); - - if u < 0.0 || u > 1.0 { continue; } - - let q = s.cross(edge1); - let v = f * ray_direction.dot(q); - - if v < 0.0 || u + v > 1.0 { continue; } - - if u + v > 1.0 { continue; } - - let t = f * edge2.dot(q); - - if t > f32::EPSILON && t < closest_t { - closest_t = t; - hit_found = true; - - let w0 = 1.0 - u - v; - let w1 = u; - let w2 = v; - - let uv0 = Vec2::from_array(v0.uv); - let uv1 = Vec2::from_array(v1.uv); - let uv2 = Vec2::from_array(v2.uv); - closest_uv = uv0 * w0 + uv1 * w1 + uv2 * w2; - - closest_local_pos = ray_origin + ray_direction * t; + geometry::raycast_uv(vertices, indices, ray_origin, ray_direction).map(|hit| { + let world_position = surface_world_matrix.transform_point3(hit.position); + let normal_matrix = world_to_mesh_local.transpose(); + let world_normal = normal_matrix.transform_vector3(hit.normal).normalize(); - let n0 = Vec3::from_array(v0.normal); - let n1 = Vec3::from_array(v1.normal); - let n2 = Vec3::from_array(v2.normal); - closest_local_normal = (n0 * w0 + n1 * w1 + n2 * w2).normalize(); + RaycastHit { + position: world_position, + normal: world_normal, + uv: hit.uv, + distance: ray_origin_world.distance(world_position), } - } - - if hit_found { - let world_position = surface_world_matrix.transform_point3(closest_local_pos); - let normal_matrix = world_to_mesh_local.transpose(); - let world_normal = normal_matrix.transform_vector3(closest_local_normal).normalize(); + }) + } - Some(RaycastHit { - uv: closest_uv, - world_position, - world_normal, - }) - } else { - None - } + pub fn record_with_transform(&self, graph: &mut Graph, surface_texture: Arc, draw_payload: &DrawPayload, transform: &Mat4) { + let image_node = graph.bind_resource(surface_texture); + self.record_with_transform_bound(graph, image_node.into(), draw_payload, transform); } - pub fn record_with_transform(&self, graph: &mut Graph, surface_texture: Arc, draw_payload: &DrawPayload, transform: Mat4) { - let push_consts = SurfacePushConstants { model_transform: transform }; + pub fn record_with_transform_bound(&self, graph: &mut Graph, surface_node: AnyImageNode, draw_payload: &DrawPayload, transform: &Mat4) { + let push_consts = SurfacePushConstants { model_transform: *transform }; let srgb_texture_view = ImageViewInfoBuilder::default() .aspect_mask(vk::ImageAspectFlags::COLOR) @@ -412,12 +353,12 @@ impl<'a> SurfaceManager<'a> { .mip_level_count(1) .build(); - let image_node = graph.bind_resource(surface_texture); let index_node = graph.bind_resource(self.scene.index_buffer.clone()); let vertex_node = graph.bind_resource(self.scene.vertex_buffer.clone()); - let index_count = self.primitive.index_count; - let first_index = self.primitive.first_index; - let base_vertex = self.primitive.base_vertex; + let prim = &self.scene.meshes[self.primitive_index[0]].primitives[self.primitive_index[1]]; + let index_count = prim.index_count; + let first_index = prim.first_index; + let base_vertex = prim.base_vertex; let mut cmd_graph = graph.begin_cmd() .debug_name("Surface") @@ -430,13 +371,13 @@ impl<'a> SurfaceManager<'a> { .resource_access(index_node, AccessType::IndexBuffer) .resource_access(vertex_node, AccessType::VertexBuffer); - cmd_graph.set_shader_subresource_access((0,1), image_node, srgb_texture_view, AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer); + cmd_graph.set_shader_subresource_access((0,1), surface_node, srgb_texture_view, AccessType::FragmentShaderReadSampledImageOrUniformTexelBuffer); cmd_graph .record_cmd(move |cmd| { cmd.bind_index_buffer(index_node, 0, IndexType::UINT32) .bind_vertex_buffer(0, vertex_node, 0) .push_constants(0, bytes_of(&push_consts)) .draw_indexed(index_count, 1, first_index, base_vertex, 0); - }).end_cmd(); + }).end_cmd(); } } \ No newline at end of file diff --git a/blender_assets/keyboard.blend b/blender_assets/keyboard.blend new file mode 100644 index 0000000..ad5c000 Binary files /dev/null and b/blender_assets/keyboard.blend differ