From 6735a0e34a15f5d5b13e407c1deae07426f71fb8 Mon Sep 17 00:00:00 2001 From: robtfm <50659922+robtfm@users.noreply.github.com> Date: Thu, 6 Jul 2023 21:24:42 +0100 Subject: [PATCH] Audio/Video (#39) - add static audio with spatial params via kira - add video playback via ffmpeg - remove macos aarch64 build due to ffmpeg issues - minor fixes - sphere radius -> 0.5 - camera mode area -> allowed on root - hopefully fix avatar race condition on startup --- .github/workflows/ci.yml | 23 ++ .github/workflows/package.yml | 27 +- Cargo.toml | 2 + crates/av/Cargo.toml | 19 ++ crates/av/src/audio_source.rs | 99 ++++++++ crates/av/src/lib.rs | 28 +++ crates/av/src/video_player.rs | 114 +++++++++ crates/av/src/video_thread.rs | 235 ++++++++++++++++++ crates/avatar/src/lib.rs | 7 +- crates/common/Cargo.toml | 2 +- crates/comms/Cargo.toml | 2 +- crates/dcl/Cargo.toml | 2 +- crates/dcl_component/build.rs | 2 + crates/dcl_component/src/lib.rs | 3 + crates/dcl_component/src/proto_components.rs | 2 + crates/ipfs/Cargo.toml | 2 +- crates/scene_runner/Cargo.toml | 2 +- .../src/update_world/camera_mode_area.rs | 2 +- .../src/update_world/gltf_container.rs | 2 +- .../scene_runner/src/update_world/material.rs | 56 ++++- .../src/update_world/mesh_renderer/mod.rs | 8 +- crates/system_ui/src/chat.rs | 4 +- crates/ui_core/src/focus.rs | 4 +- crates/ui_core/src/nine_slice.rs | 3 +- crates/ui_core/src/textentry.rs | 7 +- src/main.rs | 2 + 26 files changed, 629 insertions(+), 30 deletions(-) create mode 100644 crates/av/Cargo.toml create mode 100644 crates/av/src/audio_source.rs create mode 100644 crates/av/src/lib.rs create mode 100644 crates/av/src/video_player.rs create mode 100644 crates/av/src/video_thread.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7237ee38..1a7ac895 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,8 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install alsa and udev run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev + - name: install ffmpeg deps + run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 with: @@ -43,6 +45,23 @@ jobs: - name: Install alsa and udev run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev if: runner.os == 'linux' + - name: install ffmpeg deps (linux) + run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev + if: runner.os == 'linux' + - name: install ffmpeg deps (macOs) + if: runner.os == 'macos' + run: brew install ffmpeg pkg-config + - name: install ffmpeg deps (windows) + if: runner.os == 'windows' + run: | + $VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath) + Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n" + Invoke-WebRequest "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-full-shared.7z" -OutFile ffmpeg-release-full-shared.7z + 7z x ffmpeg-release-full-shared.7z + mkdir ffmpeg + mv ffmpeg-*/* ffmpeg/ + Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n" + Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n" - uses: actions-rs/cargo@v1 with: command: test @@ -65,6 +84,8 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install alsa and udev run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev + - name: install ffmpeg deps (linux) + run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev - uses: actions-rs/cargo@v1 with: command: fmt @@ -87,6 +108,8 @@ jobs: repo-token: ${{ secrets.GITHUB_TOKEN }} - name: Install alsa and udev run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev + - name: install ffmpeg deps (linux) + run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev - uses: actions-rs/cargo@v1 with: command: clippy diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml index 60a751bb..b3124e3c 100644 --- a/.github/workflows/package.yml +++ b/.github/workflows/package.yml @@ -49,9 +49,10 @@ jobs: os: macos-latest target: x86_64-apple-darwin - - build_name: macos-aarch64 - os: macos-latest - target: aarch64-apple-darwin +# ffmpeg does not cross-build to aarch64 in CI, so aarch64 releases must be manually added +# - build_name: macos-aarch64 +# os: macos-latest +# target: aarch64-apple-darwin - build_name: windows-x86_64 os: windows-latest @@ -85,7 +86,25 @@ jobs: - name: Install alsa and udev if: runner.os == 'linux' run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev - + + - name: install ffmpeg deps (linux) + run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev + if: runner.os == 'linux' + - name: install ffmpeg deps (macOs) + if: runner.os == 'macos' + run: brew install ffmpeg pkg-config + - name: install ffmpeg deps (windows) + if: runner.os == 'windows' + run: | + $VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath) + Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n" + Invoke-WebRequest "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-full-shared.7z" -OutFile ffmpeg-release-full-shared.7z + 7z x ffmpeg-release-full-shared.7z + mkdir ffmpeg + mv ffmpeg-*/* ffmpeg/ + Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n" + Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n" + - name: Cargo build run: cargo build --release ${{ matrix.target && '--target' }} ${{ matrix.target }} env: diff --git a/Cargo.toml b/Cargo.toml index ad98c1c4..46a32d13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ visuals = { path="crates/visuals" } ui_core = { path="crates/ui_core" } scene_runner = { path="crates/scene_runner" } console = { path="crates/console" } +av = { path="crates/av" } bevy = "0.10" bevy_console = "0.7.0" @@ -39,4 +40,5 @@ prost-build = "0.11.8" [patch.crates-io] bevy = { git="https://github.com/robtfm/bevy", branch="0.10-hotfix" } +# bevy = { path = "../bevy" } bevy_mod_billboard = { git="https://github.com/robtfm/bevy_mod_billboard", branch="lock_all" } diff --git a/crates/av/Cargo.toml b/crates/av/Cargo.toml new file mode 100644 index 00000000..f5955ec8 --- /dev/null +++ b/crates/av/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "av" +version = "0.1.0" +edition = "2021" + +[lib] + +[dependencies] +common = { path = "../common" } +dcl = { path="../dcl" } +dcl_component = { path="../dcl_component" } +ipfs = { path="../ipfs" } +scene_runner = { path="../scene_runner" } + +bevy = "0.10" +bevy_kira_audio = { version= "0.15", features=["flac", "mp3", "ogg", "wav"] } +ffmpeg-next = "6.0.0" +tokio = { version = "1.29.1", features = ["sync"] } +anyhow = "1.0.70" \ No newline at end of file diff --git a/crates/av/src/audio_source.rs b/crates/av/src/audio_source.rs new file mode 100644 index 00000000..3eec4ac9 --- /dev/null +++ b/crates/av/src/audio_source.rs @@ -0,0 +1,99 @@ +use std::time::Duration; + +use bevy::prelude::*; +use bevy_kira_audio::{ + prelude::{AudioEmitter, AudioReceiver}, + AudioControl, AudioInstance, AudioTween, +}; +use common::{structs::PrimaryCameraRes, util::TryInsertEx}; +use dcl_component::proto_components::sdk::components::PbAudioSource; +use ipfs::IpfsLoaderExt; +use scene_runner::{renderer_context::RendererSceneContext, SceneEntity}; + +#[derive(Component, Debug)] +pub struct AudioSource(PbAudioSource); + +impl From for AudioSource { + fn from(value: PbAudioSource) -> Self { + Self(value) + } +} + +pub(crate) fn setup_audio(mut commands: Commands, camera: Res) { + commands.entity(camera.0).try_insert(AudioReceiver); +} + +#[allow(clippy::type_complexity)] +pub(crate) fn update_audio( + mut commands: Commands, + mut query: Query< + ( + Entity, + &SceneEntity, + &AudioSource, + Option<&Handle>, + Option<&mut AudioEmitter>, + ), + Changed, + >, + scenes: Query<&RendererSceneContext>, + audio: Res, + asset_server: Res, + mut audio_instances: ResMut>, +) { + for (ent, scene_ent, audio_source, maybe_source, maybe_emitter) in query.iter_mut() { + // preload clips + let h_audio = match maybe_source { + Some(instance) => instance.clone_weak(), + None => { + let Ok(scene) = scenes.get(scene_ent.root) else { + warn!("failed to load audio source scene"); + continue; + }; + + let Ok(handle) = asset_server.load_content_file(&audio_source.0.audio_clip_url, &scene.hash) else { + warn!("failed to load content file"); + continue; + }; + + let h_audio = handle.clone_weak(); + commands.entity(ent).try_insert(handle); + h_audio + } + }; + + if audio_source.0.playing() { + // stop previous - is this right? + if let Some(emitter) = maybe_emitter { + for h_instance in emitter.instances.iter() { + if let Some(instance) = audio_instances.get_mut(h_instance) { + instance.stop(AudioTween::linear(Duration::ZERO)); + } + } + } + + let mut instance = &mut audio.play(h_audio); + if audio_source.0.r#loop() { + instance = instance.looped(); + } + + if let Some(volume) = audio_source.0.volume { + instance = instance + .with_volume(bevy_kira_audio::prelude::Volume::Amplitude(volume as f64)); + } + + let instance = instance.handle(); + commands.entity(ent).try_insert(AudioEmitter { + instances: vec![instance], + }); + } else if let Some(mut emitter) = maybe_emitter { + // stop running + for h_instance in emitter.instances.iter() { + if let Some(instance) = audio_instances.get_mut(h_instance) { + instance.stop(AudioTween::linear(Duration::ZERO)); + } + } + emitter.instances.clear(); + } + } +} diff --git a/crates/av/src/lib.rs b/crates/av/src/lib.rs new file mode 100644 index 00000000..e176e6b7 --- /dev/null +++ b/crates/av/src/lib.rs @@ -0,0 +1,28 @@ +pub mod audio_source; +pub mod video_player; +pub mod video_thread; + +use audio_source::{setup_audio, update_audio}; +use bevy::prelude::*; +use bevy_kira_audio::prelude::SpacialAudio; +use common::sets::{SceneSets, SetupSets}; +use dcl::interface::ComponentPosition; +use dcl_component::{proto_components::sdk::components::PbAudioSource, SceneComponentId}; +use scene_runner::update_world::AddCrdtInterfaceExt; +use video_player::VideoPlayerPlugin; + +pub struct AudioPlugin; + +impl Plugin for AudioPlugin { + fn build(&self, app: &mut App) { + app.add_plugin(bevy_kira_audio::AudioPlugin); + app.add_plugin(VideoPlayerPlugin); + app.add_crdt_lww_component::( + SceneComponentId::AUDIO_SOURCE, + ComponentPosition::EntityOnly, + ); + app.add_system(update_audio.in_set(SceneSets::PostLoop)); + app.insert_resource(SpacialAudio { max_distance: 25. }); + app.add_startup_system(setup_audio.in_set(SetupSets::Main)); + } +} diff --git a/crates/av/src/video_player.rs b/crates/av/src/video_player.rs new file mode 100644 index 00000000..591800a4 --- /dev/null +++ b/crates/av/src/video_player.rs @@ -0,0 +1,114 @@ +// TODO: rate, audio channels, update position(?) + +use bevy::{ + prelude::*, + render::render_resource::{Extent3d, TextureDimension, TextureFormat, TextureUsages}, +}; +use common::{sets::SceneSets, util::TryInsertEx}; +use dcl::interface::ComponentPosition; +use dcl_component::{proto_components::sdk::components::PbVideoPlayer, SceneComponentId}; +use scene_runner::update_world::{material::VideoTextureOutput, AddCrdtInterfaceExt}; + +use crate::video_thread::{VideoCommand, VideoData, VideoInfo, VideoSink}; + +pub struct VideoPlayerPlugin; + +impl Plugin for VideoPlayerPlugin { + fn build(&self, app: &mut App) { + app.add_crdt_lww_component::( + SceneComponentId::VIDEO_PLAYER, + ComponentPosition::EntityOnly, + ); + app.add_startup_system(init_ffmpeg); + app.add_system(play_videos); + app.add_system(update_video_players.in_set(SceneSets::PostLoop)); + } +} + +#[derive(Component)] +pub struct VideoPlayer(pub PbVideoPlayer); + +impl From for VideoPlayer { + fn from(value: PbVideoPlayer) -> Self { + Self(value) + } +} + +fn init_ffmpeg() { + ffmpeg_next::init().unwrap(); + ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Error); +} + +fn play_videos(mut images: ResMut>, mut q: Query<&mut VideoSink>) { + for mut sink in q.iter_mut() { + match sink.data_receiver.try_recv() { + Ok(VideoData::Info(VideoInfo { + width, + height, + rate, + length, + })) => { + debug!("resize"); + images.get_mut(&sink.image).unwrap().resize(Extent3d { + width, + height, + depth_or_array_layers: 1, + }); + sink.length = Some(length); + sink.rate = Some(rate); + } + Ok(VideoData::Frame(frame, time)) => { + debug!("set frame on {:?}", sink.image); + images + .get_mut(&sink.image) + .unwrap() + .data + .copy_from_slice(frame.data(0)); + sink.current_time = time; + } + Err(_) => (), + } + } +} + +pub fn update_video_players( + mut commands: Commands, + video_players: Query<(Entity, &VideoPlayer, Option<&VideoSink>), Changed>, + mut images: ResMut>, +) { + for (ent, player, maybe_sink) in video_players.iter() { + if maybe_sink.map(|sink| &sink.source) != Some(&player.0.src) { + let mut image = Image::new_fill( + bevy::render::render_resource::Extent3d { + width: 8, + height: 8, + depth_or_array_layers: 1, + }, + TextureDimension::D2, + &Color::PINK.as_rgba_u32().to_le_bytes(), + TextureFormat::Rgba8UnormSrgb, + ); + image.texture_descriptor.usage = + TextureUsages::COPY_DST | TextureUsages::TEXTURE_BINDING; + let image_handle = images.add(image); + let video_sink = VideoSink::new( + player.0.src.clone(), + image_handle, + player.0.playing.unwrap_or(true), + player.0.r#loop.unwrap_or(false), + ); + let video_output = VideoTextureOutput(video_sink.image.clone()); + commands.entity(ent).try_insert((video_sink, video_output)); + } else { + let sink = maybe_sink.as_ref().unwrap(); + if player.0.playing.unwrap_or(true) { + let _ = sink.command_sender.blocking_send(VideoCommand::Play); + } else { + let _ = sink.command_sender.blocking_send(VideoCommand::Pause); + } + let _ = sink + .command_sender + .blocking_send(VideoCommand::Repeat(player.0.r#loop.unwrap_or(false))); + } + } +} diff --git a/crates/av/src/video_thread.rs b/crates/av/src/video_thread.rs new file mode 100644 index 00000000..8747c0dc --- /dev/null +++ b/crates/av/src/video_thread.rs @@ -0,0 +1,235 @@ +use std::time::{Duration, Instant}; + +use bevy::prelude::*; +use ffmpeg_next::frame::Video; +use ffmpeg_next::media::Type; +use ffmpeg_next::software::scaling::{context::Context, flag::Flags}; +use ffmpeg_next::{ + ffi::AV_TIME_BASE, + format::{input, Pixel}, +}; +use tokio::sync::mpsc::error::TryRecvError; + +pub enum VideoCommand { + Play, + Pause, + Repeat(bool), + Seek(f64), + Dispose, +} + +pub struct VideoInfo { + pub width: u32, + pub height: u32, + pub rate: f64, + pub length: f64, +} + +pub enum VideoData { + Info(VideoInfo), + Frame(Video, f64), +} + +#[derive(Component)] +pub struct VideoSink { + pub source: String, + pub command_sender: tokio::sync::mpsc::Sender, + pub data_receiver: tokio::sync::mpsc::Receiver, + pub image: Handle, + pub current_time: f64, + pub length: Option, + pub rate: Option, +} + +impl VideoSink { + pub fn new(source: String, image: Handle, playing: bool, repeat: bool) -> Self { + let (command_sender, command_receiver) = tokio::sync::mpsc::channel(10); + let (data_sender, data_receiver) = tokio::sync::mpsc::channel(10); + + spawn_video_thread(command_receiver, data_sender, source.clone()); + + if playing { + command_sender.blocking_send(VideoCommand::Play).unwrap(); + } + command_sender + .blocking_send(VideoCommand::Repeat(repeat)) + .unwrap(); + + Self { + source, + command_sender, + data_receiver, + image, + current_time: 0.0, + length: None, + rate: None, + } + } +} + +pub fn spawn_video_thread( + commands: tokio::sync::mpsc::Receiver, + frames: tokio::sync::mpsc::Sender, + path: String, +) { + std::thread::spawn(move || video_thread(commands, frames, path)); +} + +fn video_thread( + commands: tokio::sync::mpsc::Receiver, + frames: tokio::sync::mpsc::Sender, + path: String, +) { + if let Err(e) = video_thread_inner(commands, frames, path) { + warn!("video error: {e}"); + } else { + debug!("video closed"); + } +} + +pub fn video_thread_inner( + mut commands: tokio::sync::mpsc::Receiver, + sink: tokio::sync::mpsc::Sender, + path: String, +) -> Result<(), anyhow::Error> { + let mut input_context = input(&path)?; + + // initialize decoder + let input_stream = input_context + .streams() + .best(Type::Video) + .ok_or(ffmpeg_next::Error::StreamNotFound)?; + + let video_stream_index = input_stream.index(); + + let context_decoder = + ffmpeg_next::codec::context::Context::from_parameters(input_stream.parameters())?; + + let mut decoder = context_decoder.decoder().video()?; + + let roundup = |x: u32| { + (x.saturating_sub(1) / 8 + 1) * 8 + // x + }; + + // initialize scaler + let mut scaler_context = Context::get( + decoder.format(), + decoder.width(), + decoder.height(), + Pixel::RGBA, + roundup(decoder.width()), + roundup(decoder.height()), + Flags::BILINEAR, + )?; + + let rate = f64::from(input_stream.rate()); + let length = (input_stream.frames() as f64) / rate; + debug!("frames: {}, length: {}", input_stream.frames(), length); + + if sink + .blocking_send(VideoData::Info(VideoInfo { + width: roundup(decoder.width()), + height: roundup(decoder.height()), + rate, + length, + })) + .is_err() + { + // channel closed + return Ok(()); + } + + let mut start_frame = 0; + let mut current_frame = 0; + let mut start_instant: Option = None; + let mut repeat = false; + let mut next_frame = None; + + loop { + if next_frame.is_none() { + while let Some((stream, packet)) = input_context.packets().next() { + // check if packets is for the selected video stream + if stream.index() == video_stream_index { + decoder.send_packet(&packet).unwrap(); + let mut decoded = Video::empty(); + if let Ok(()) = decoder.receive_frame(&mut decoded) { + let mut rgb_frame = Video::empty(); + // run frame through scaler for color space conversion + scaler_context.run(&decoded, &mut rgb_frame)?; + next_frame = Some(rgb_frame); + break; + } + } + } + } + + if next_frame.is_none() { + // eof + if repeat { + if input_context.seek(0, ..).is_err() { + // reload + input_context = input(&path)?; + } + start_frame = 0; + current_frame = 0; + if start_instant.is_some() { + start_instant = Some(Instant::now()); + } + continue; + } else { + start_instant = None; + } + } + + let cmd = if start_instant.is_some() { + commands.try_recv() + } else { + commands.blocking_recv().ok_or(TryRecvError::Disconnected) + }; + + match cmd { + Ok(VideoCommand::Play) => { + if start_instant.is_none() { + start_instant = Some(Instant::now()) + } + } + Ok(VideoCommand::Pause) => start_instant = None, + Ok(VideoCommand::Repeat(r)) => repeat = r, + Ok(VideoCommand::Seek(time)) => { + debug!("seek -> {time}"); + let time = time.clamp(0.0, length); + let av_time = (time * i64::from(AV_TIME_BASE) as f64) as i64; + input_context.seek(av_time, 0..)?; + let frame = (time * rate) as i64; + start_frame = frame; + current_frame = frame; + if start_instant.is_some() { + start_instant = Some(Instant::now()); + } + } + Err(TryRecvError::Empty) => (), + Err(TryRecvError::Disconnected) | Ok(VideoCommand::Dispose) => return Ok(()), + } + + if let Some(play_instant) = start_instant { + let now = Instant::now(); + let next_frame_time = play_instant + + Duration::from_secs_f64((current_frame - start_frame + 1) as f64 / rate); + if let Some(sleep_time) = next_frame_time.checked_duration_since(now) { + std::thread::sleep(sleep_time); + } else { + // we lost time - reset start frame and instant + start_frame = current_frame + 1; + start_instant = Some(now); + } + + current_frame += 1; + debug!("send frame {current_frame}"); + let _ = sink.blocking_send(VideoData::Frame( + next_frame.take().unwrap(), + current_frame as f64 / rate, + )); + } + } +} diff --git a/crates/avatar/src/lib.rs b/crates/avatar/src/lib.rs index 3396be0f..890a134e 100644 --- a/crates/avatar/src/lib.rs +++ b/crates/avatar/src/lib.rs @@ -1018,9 +1018,10 @@ fn spawn_scenes( .and_then(|h_gltf| gltfs.get(h_gltf)) else { match def.body.model.as_ref().map(|h_gtlf| asset_server.get_load_state(h_gtlf)) { - Some(bevy::asset::LoadState::Loading) => (), - _ => { - warn!("failed to load body gltf"); + Some(bevy::asset::LoadState::Loading) | + Some(bevy::asset::LoadState::NotLoaded) => (), + otherwise => { + warn!("failed to load body gltf: {otherwise:?}"); commands.entity(ent).try_insert(AvatarProcessed); } } diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index aa7c2f0b..bae9433c 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -10,5 +10,5 @@ bevy = "0.10" ethers = "2.0.3" futures-lite = "1.12.0" hex = "0.4.3" -tokio = { version = "1.26.0", features = ["sync"] } +tokio = { version = "1.29.1", features = ["sync"] } serde = "1.0.152" diff --git a/crates/comms/Cargo.toml b/crates/comms/Cargo.toml index 9133cb2e..00594d25 100644 --- a/crates/comms/Cargo.toml +++ b/crates/comms/Cargo.toml @@ -14,7 +14,7 @@ ipfs = { path="../ipfs" } bevy = "0.10" bimap = "0.6.3" ethers = "2.0.3" -tokio = { version = "1.26.0", features = ["sync"] } +tokio = { version = "1.29.1", features = ["sync"] } serde = "1.0.152" serde_json = { version = "1.0.92", features = ["raw_value"] } async-trait = "0.1.68" diff --git a/crates/dcl/Cargo.toml b/crates/dcl/Cargo.toml index 4c028681..6471b3b1 100644 --- a/crates/dcl/Cargo.toml +++ b/crates/dcl/Cargo.toml @@ -11,7 +11,7 @@ dcl_component = { path="../dcl_component" } ipfs = { path="../ipfs" } bevy = "0.10" -tokio = { version = "1.26.0", features = ["sync"] } +tokio = { version = "1.29.1", features = ["sync"] } once_cell = "1.16.0" deno_core = "0.183.*" futures-lite = "1.12.0" diff --git a/crates/dcl_component/build.rs b/crates/dcl_component/build.rs index b3b0936e..e95e86c3 100644 --- a/crates/dcl_component/build.rs +++ b/crates/dcl_component/build.rs @@ -31,6 +31,8 @@ fn main() -> Result<()> { "pointer_lock", "camera_mode", "camera_mode_area", + "audio_source", + "video_player", ]; let mut sources = components diff --git a/crates/dcl_component/src/lib.rs b/crates/dcl_component/src/lib.rs index a61c0ad5..371db276 100644 --- a/crates/dcl_component/src/lib.rs +++ b/crates/dcl_component/src/lib.rs @@ -58,12 +58,15 @@ impl SceneComponentId { pub const MATERIAL: SceneComponentId = SceneComponentId(1017); pub const MESH_RENDERER: SceneComponentId = SceneComponentId(1018); pub const MESH_COLLIDER: SceneComponentId = SceneComponentId(1019); + pub const AUDIO_SOURCE: SceneComponentId = SceneComponentId(1020); pub const TEXT_SHAPE: SceneComponentId = SceneComponentId(1030); pub const GLTF_CONTAINER: SceneComponentId = SceneComponentId(1041); pub const ANIMATOR: SceneComponentId = SceneComponentId(1042); + pub const VIDEO_PLAYER: SceneComponentId = SceneComponentId(1043); + pub const ENGINE_INFO: SceneComponentId = SceneComponentId(1048); pub const GLTF_CONTAINER_LOADING_STATE: SceneComponentId = SceneComponentId(1049); diff --git a/crates/dcl_component/src/proto_components.rs b/crates/dcl_component/src/proto_components.rs index 160693a2..1464dec4 100644 --- a/crates/dcl_component/src/proto_components.rs +++ b/crates/dcl_component/src/proto_components.rs @@ -84,6 +84,8 @@ impl DclProtoComponent for sdk::components::PbTextShape {} impl DclProtoComponent for sdk::components::PbPointerLock {} impl DclProtoComponent for sdk::components::PbCameraMode {} impl DclProtoComponent for sdk::components::PbCameraModeArea {} +impl DclProtoComponent for sdk::components::PbAudioSource {} +impl DclProtoComponent for sdk::components::PbVideoPlayer {} // VECTOR3 conversions impl Copy for common::Vector3 {} diff --git a/crates/ipfs/Cargo.toml b/crates/ipfs/Cargo.toml index 43465fbf..6ac19765 100644 --- a/crates/ipfs/Cargo.toml +++ b/crates/ipfs/Cargo.toml @@ -16,7 +16,7 @@ bimap = "0.6.3" isahc = { version = "1.7.2", features = ["json"] } serde = "1.0.152" serde_json = { version = "1.0.92", features = ["raw_value"] } -tokio = { version = "1.26.0", features = ["sync"] } +tokio = { version = "1.29.1", features = ["sync"] } clap = "4.1.10" urlencoding = "2.1.2" urn = "0.7.0" diff --git a/crates/scene_runner/Cargo.toml b/crates/scene_runner/Cargo.toml index 5a40d66b..b4732d64 100644 --- a/crates/scene_runner/Cargo.toml +++ b/crates/scene_runner/Cargo.toml @@ -16,7 +16,7 @@ console = { path="../console" } input_manager = { path="../input_manager" } bevy = { version = "0.10", features=["jpeg"] } -tokio = { version = "1.26.0", features = ["sync"] } +tokio = { version = "1.29.1", features = ["sync"] } serde = "1.0.152" serde_json = { version = "1.0.92", features = ["raw_value"] } anyhow = "1.0.70" diff --git a/crates/scene_runner/src/update_world/camera_mode_area.rs b/crates/scene_runner/src/update_world/camera_mode_area.rs index 6eb96a60..5d03ea21 100644 --- a/crates/scene_runner/src/update_world/camera_mode_area.rs +++ b/crates/scene_runner/src/update_world/camera_mode_area.rs @@ -30,7 +30,7 @@ impl Plugin for CameraModeAreaPlugin { fn build(&self, app: &mut App) { app.add_crdt_lww_component::( SceneComponentId::CAMERA_MODE_AREA, - ComponentPosition::EntityOnly, + ComponentPosition::Any, ); app.add_system(update_camera_mode_area.in_set(SceneSets::PostLoop)); diff --git a/crates/scene_runner/src/update_world/gltf_container.rs b/crates/scene_runner/src/update_world/gltf_container.rs index 4a718490..9c117abf 100644 --- a/crates/scene_runner/src/update_world/gltf_container.rs +++ b/crates/scene_runner/src/update_world/gltf_container.rs @@ -329,7 +329,7 @@ fn update_gltf( let is_skinned = mesh_data.attribute(Mesh::ATTRIBUTE_JOINT_WEIGHT).is_some(); if is_skinned { // bevy doesn't calculate culling correctly for skinned entities - commands.entity(spawned_ent).insert(NoFrustumCulling); + commands.entity(spawned_ent).try_insert(NoFrustumCulling); } let mut collider_base_name = diff --git a/crates/scene_runner/src/update_world/material.rs b/crates/scene_runner/src/update_world/material.rs index 8af88eef..76d57e64 100644 --- a/crates/scene_runner/src/update_world/material.rs +++ b/crates/scene_runner/src/update_world/material.rs @@ -5,10 +5,10 @@ use common::util::TryInsertEx; use dcl::interface::ComponentPosition; use dcl_component::{ proto_components::{ - common::{texture_union, TextureUnion}, + common::{texture_union, TextureUnion, VideoTexture}, sdk::components::{pb_material, MaterialTransparencyMode, PbMaterial}, }, - SceneComponentId, + SceneComponentId, SceneEntityId, }; use ipfs::IpfsLoaderExt; @@ -116,15 +116,27 @@ impl Plugin for MaterialDefinitionPlugin { } } +#[derive(Component)] +pub struct RetryMaterial; + +#[derive(Component)] +pub struct TouchMaterial; + +#[derive(Component)] +pub struct VideoTextureOutput(pub Handle); + +#[allow(clippy::type_complexity)] fn update_materials( mut commands: Commands, mut new_materials: Query< (Entity, &MaterialDefinition, &ContainerEntity), - Changed, + Or<(Changed, With)>, >, mut materials: ResMut>, asset_server: Res, scenes: Query<&RendererSceneContext>, + videos: Query<&VideoTextureOutput>, + touch: Query<&Handle, With>, ) { for (ent, defn, container) in new_materials.iter_mut() { // get texture @@ -138,20 +150,50 @@ fn update_materials( .load_content_file::(&texture.src, &root.hash) .ok() }) + } else if let Some(TextureUnion { + tex: + Some(texture_union::Tex::VideoTexture(VideoTexture { + video_player_entity, + .. + })), + }) = defn.base_color_texture.as_ref() + { + let Some(video_entity) = scenes.get(container.root).ok().and_then(|root| { + root.bevy_entity(SceneEntityId::from_proto_u32(*video_player_entity)) + }) else { + warn!("failed to look up video source entity"); + continue; + }; + + if let Ok(vt) = videos.get(video_entity) { + debug!("adding video texture {:?}", vt.0); + commands.entity(ent).insert(TouchMaterial); + Some(vt.0.clone()) + } else { + warn!("video source entity not ready, retrying ..."); + commands.entity(ent).insert(RetryMaterial); + continue; + } } else { None }; // info!("found a mat for {ent:?}"); let mut commands = commands.entity(ent); - commands.try_insert(materials.add(StandardMaterial { - base_color_texture, - ..defn.material.clone() - })); + commands + .remove::() + .try_insert(materials.add(StandardMaterial { + base_color_texture, + ..defn.material.clone() + })); if defn.shadow_caster { commands.remove::(); } else { commands.try_insert(NotShadowCaster); } } + + for touch in touch.iter() { + materials.get_mut(touch); + } } diff --git a/crates/scene_runner/src/update_world/mesh_renderer/mod.rs b/crates/scene_runner/src/update_world/mesh_renderer/mod.rs index a4352089..75bcc76e 100644 --- a/crates/scene_runner/src/update_world/mesh_renderer/mod.rs +++ b/crates/scene_runner/src/update_world/mesh_renderer/mod.rs @@ -74,7 +74,13 @@ impl Plugin for MeshDefinitionPlugin { let boxx = assets.add(shape::Cube::default().into()); let cylinder = assets.add(shape::Cylinder::default().into()); let plane = assets.add(shape::Quad::default().into()); - let sphere = assets.add(shape::UVSphere::default().into()); + let sphere = assets.add( + shape::UVSphere { + radius: 0.5, + ..Default::default() + } + .into(), + ); app.insert_resource(MeshPrimitiveDefaults { boxx, plane, diff --git a/crates/system_ui/src/chat.rs b/crates/system_ui/src/chat.rs index b04f0a12..ec23a7c1 100644 --- a/crates/system_ui/src/chat.rs +++ b/crates/system_ui/src/chat.rs @@ -7,7 +7,7 @@ use common::{ dcl_assert, sets::SetupSets, structs::PrimaryUser, - util::{RingBuffer, RingBufferReceiver}, + util::{RingBuffer, RingBufferReceiver, TryInsertEx}, }; use comms::{global_crdt::ChatEvent, profile::UserProfile, NetworkMessage, Transport}; use dcl::{SceneLogLevel, SceneLogMessage}; @@ -114,7 +114,7 @@ fn setup(mut commands: Commands, asset_server: Res, root: Res::new(update_chatbox_focus), On::::new( |mut commands: Commands, q: Query>| { - commands.entity(q.single()).insert(Focus); + commands.entity(q.single()).try_insert(Focus); }, ), )) diff --git a/crates/ui_core/src/focus.rs b/crates/ui_core/src/focus.rs index 5a8c566f..61daa718 100644 --- a/crates/ui_core/src/focus.rs +++ b/crates/ui_core/src/focus.rs @@ -1,7 +1,7 @@ use bevy::prelude::*; use super::ui_actions::UiActionSet; -use common::sets::SceneSets; +use common::{sets::SceneSets, util::TryInsertEx}; #[derive(Component)] pub struct Focus; @@ -48,6 +48,6 @@ fn focus( .iter() .filter(|(_, interaction)| matches!(interaction, Interaction::Clicked)) { - commands.entity(entity).insert(Focus); + commands.entity(entity).try_insert(Focus); } } diff --git a/crates/ui_core/src/nine_slice.rs b/crates/ui_core/src/nine_slice.rs index 2ae62ab5..906d7bd1 100644 --- a/crates/ui_core/src/nine_slice.rs +++ b/crates/ui_core/src/nine_slice.rs @@ -1,4 +1,5 @@ use bevy::prelude::*; +use common::util::TryInsertEx; /// specify a background image using 9-slice scaling /// https://en.wikipedia.org/wiki/9-slice_scaling @@ -154,7 +155,7 @@ fn update_slices( // get or build tree let Some(container) = maybe_children.and_then(|children| children.iter().find(|child| existing_slices.get(**child).is_ok())) else { // build - commands.entity(ent).insert(SliceInitMarker).with_children(|c| { + commands.entity(ent).try_insert(SliceInitMarker).with_children(|c| { // container c.spawn(( NodeBundle{ diff --git a/crates/ui_core/src/textentry.rs b/crates/ui_core/src/textentry.rs index c834d4a2..15368a87 100644 --- a/crates/ui_core/src/textentry.rs +++ b/crates/ui_core/src/textentry.rs @@ -3,6 +3,7 @@ use bevy_egui::{ egui::{self, TextEdit}, EguiContext, }; +use common::util::TryInsertEx; use crate::ui_actions::DataChanged; @@ -90,7 +91,7 @@ pub fn update_text_entry_components( ); if response.changed() && !textbox.accept_line { - commands.entity(entity).insert(DataChanged); + commands.entity(entity).try_insert(DataChanged); } // pass through focus and interaction @@ -100,7 +101,7 @@ pub fn update_text_entry_components( let message = std::mem::take(&mut textbox.content); response.request_focus(); textbox.messages.push(message); - commands.entity(entity).insert(DataChanged); + commands.entity(entity).try_insert(DataChanged); } else { commands.entity(entity).remove::(); defocus = true; @@ -125,7 +126,7 @@ pub fn update_text_entry_components( response.surrender_focus() } else if response.has_focus() { debug!("tb focus -> Focus"); - commands.entity(entity).insert(Focus); + commands.entity(entity).try_insert(Focus); } } }); diff --git a/src/main.rs b/src/main.rs index 95a70063..8999ff0f 100644 --- a/src/main.rs +++ b/src/main.rs @@ -22,6 +22,7 @@ use scene_runner::{ SceneRunnerPlugin, }; +use av::AudioPlugin; use avatar::AvatarPlugin; use comms::{wallet::WalletPlugin, CommsPlugin}; use console::{ConsolePlugin, DoAddConsoleCommand}; @@ -159,6 +160,7 @@ fn main() { .add_plugin(WalletPlugin) .add_plugin(CommsPlugin) .add_plugin(AvatarPlugin) + .add_plugin(AudioPlugin) .insert_resource(PrimaryCameraRes(Entity::PLACEHOLDER)) .add_startup_system(setup.in_set(SetupSets::Init)) .insert_resource(AmbientLight {