-
Notifications
You must be signed in to change notification settings - Fork 10
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- add static audio with spatial params via kira - add video playback via ffmpeg - remove macos aarch64 build due to ffmpeg issues - minor fixes - sphere radius -> 0.5 - camera mode area -> allowed on root - hopefully fix avatar race condition on startup
- Loading branch information
Showing
26 changed files
with
629 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| [package] | ||
| name = "av" | ||
| version = "0.1.0" | ||
| edition = "2021" | ||
|
|
||
| [lib] | ||
|
|
||
| [dependencies] | ||
| common = { path = "../common" } | ||
| dcl = { path="../dcl" } | ||
| dcl_component = { path="../dcl_component" } | ||
| ipfs = { path="../ipfs" } | ||
| scene_runner = { path="../scene_runner" } | ||
|
|
||
| bevy = "0.10" | ||
| bevy_kira_audio = { version= "0.15", features=["flac", "mp3", "ogg", "wav"] } | ||
| ffmpeg-next = "6.0.0" | ||
| tokio = { version = "1.29.1", features = ["sync"] } | ||
| anyhow = "1.0.70" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,99 @@ | ||
| use std::time::Duration; | ||
|
|
||
| use bevy::prelude::*; | ||
| use bevy_kira_audio::{ | ||
| prelude::{AudioEmitter, AudioReceiver}, | ||
| AudioControl, AudioInstance, AudioTween, | ||
| }; | ||
| use common::{structs::PrimaryCameraRes, util::TryInsertEx}; | ||
| use dcl_component::proto_components::sdk::components::PbAudioSource; | ||
| use ipfs::IpfsLoaderExt; | ||
| use scene_runner::{renderer_context::RendererSceneContext, SceneEntity}; | ||
|
|
||
| #[derive(Component, Debug)] | ||
| pub struct AudioSource(PbAudioSource); | ||
|
|
||
| impl From<PbAudioSource> for AudioSource { | ||
| fn from(value: PbAudioSource) -> Self { | ||
| Self(value) | ||
| } | ||
| } | ||
|
|
||
| pub(crate) fn setup_audio(mut commands: Commands, camera: Res<PrimaryCameraRes>) { | ||
| commands.entity(camera.0).try_insert(AudioReceiver); | ||
| } | ||
|
|
||
| #[allow(clippy::type_complexity)] | ||
| pub(crate) fn update_audio( | ||
| mut commands: Commands, | ||
| mut query: Query< | ||
| ( | ||
| Entity, | ||
| &SceneEntity, | ||
| &AudioSource, | ||
| Option<&Handle<bevy_kira_audio::AudioSource>>, | ||
| Option<&mut AudioEmitter>, | ||
| ), | ||
| Changed<AudioSource>, | ||
| >, | ||
| scenes: Query<&RendererSceneContext>, | ||
| audio: Res<bevy_kira_audio::Audio>, | ||
| asset_server: Res<AssetServer>, | ||
| mut audio_instances: ResMut<Assets<AudioInstance>>, | ||
| ) { | ||
| for (ent, scene_ent, audio_source, maybe_source, maybe_emitter) in query.iter_mut() { | ||
| // preload clips | ||
| let h_audio = match maybe_source { | ||
| Some(instance) => instance.clone_weak(), | ||
| None => { | ||
| let Ok(scene) = scenes.get(scene_ent.root) else { | ||
| warn!("failed to load audio source scene"); | ||
| continue; | ||
| }; | ||
|
|
||
| let Ok(handle) = asset_server.load_content_file(&audio_source.0.audio_clip_url, &scene.hash) else { | ||
| warn!("failed to load content file"); | ||
| continue; | ||
| }; | ||
|
|
||
| let h_audio = handle.clone_weak(); | ||
| commands.entity(ent).try_insert(handle); | ||
| h_audio | ||
| } | ||
| }; | ||
|
|
||
| if audio_source.0.playing() { | ||
| // stop previous - is this right? | ||
| if let Some(emitter) = maybe_emitter { | ||
| for h_instance in emitter.instances.iter() { | ||
| if let Some(instance) = audio_instances.get_mut(h_instance) { | ||
| instance.stop(AudioTween::linear(Duration::ZERO)); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| let mut instance = &mut audio.play(h_audio); | ||
| if audio_source.0.r#loop() { | ||
| instance = instance.looped(); | ||
| } | ||
|
|
||
| if let Some(volume) = audio_source.0.volume { | ||
| instance = instance | ||
| .with_volume(bevy_kira_audio::prelude::Volume::Amplitude(volume as f64)); | ||
| } | ||
|
|
||
| let instance = instance.handle(); | ||
| commands.entity(ent).try_insert(AudioEmitter { | ||
| instances: vec![instance], | ||
| }); | ||
| } else if let Some(mut emitter) = maybe_emitter { | ||
| // stop running | ||
| for h_instance in emitter.instances.iter() { | ||
| if let Some(instance) = audio_instances.get_mut(h_instance) { | ||
| instance.stop(AudioTween::linear(Duration::ZERO)); | ||
| } | ||
| } | ||
| emitter.instances.clear(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| pub mod audio_source; | ||
| pub mod video_player; | ||
| pub mod video_thread; | ||
|
|
||
| use audio_source::{setup_audio, update_audio}; | ||
| use bevy::prelude::*; | ||
| use bevy_kira_audio::prelude::SpacialAudio; | ||
| use common::sets::{SceneSets, SetupSets}; | ||
| use dcl::interface::ComponentPosition; | ||
| use dcl_component::{proto_components::sdk::components::PbAudioSource, SceneComponentId}; | ||
| use scene_runner::update_world::AddCrdtInterfaceExt; | ||
| use video_player::VideoPlayerPlugin; | ||
|
|
||
| pub struct AudioPlugin; | ||
|
|
||
| impl Plugin for AudioPlugin { | ||
| fn build(&self, app: &mut App) { | ||
| app.add_plugin(bevy_kira_audio::AudioPlugin); | ||
| app.add_plugin(VideoPlayerPlugin); | ||
| app.add_crdt_lww_component::<PbAudioSource, audio_source::AudioSource>( | ||
| SceneComponentId::AUDIO_SOURCE, | ||
| ComponentPosition::EntityOnly, | ||
| ); | ||
| app.add_system(update_audio.in_set(SceneSets::PostLoop)); | ||
| app.insert_resource(SpacialAudio { max_distance: 25. }); | ||
| app.add_startup_system(setup_audio.in_set(SetupSets::Main)); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| // TODO: rate, audio channels, update position(?) | ||
|
|
||
| use bevy::{ | ||
| prelude::*, | ||
| render::render_resource::{Extent3d, TextureDimension, TextureFormat, TextureUsages}, | ||
| }; | ||
| use common::{sets::SceneSets, util::TryInsertEx}; | ||
| use dcl::interface::ComponentPosition; | ||
| use dcl_component::{proto_components::sdk::components::PbVideoPlayer, SceneComponentId}; | ||
| use scene_runner::update_world::{material::VideoTextureOutput, AddCrdtInterfaceExt}; | ||
|
|
||
| use crate::video_thread::{VideoCommand, VideoData, VideoInfo, VideoSink}; | ||
|
|
||
| pub struct VideoPlayerPlugin; | ||
|
|
||
| impl Plugin for VideoPlayerPlugin { | ||
| fn build(&self, app: &mut App) { | ||
| app.add_crdt_lww_component::<PbVideoPlayer, VideoPlayer>( | ||
| SceneComponentId::VIDEO_PLAYER, | ||
| ComponentPosition::EntityOnly, | ||
| ); | ||
| app.add_startup_system(init_ffmpeg); | ||
| app.add_system(play_videos); | ||
| app.add_system(update_video_players.in_set(SceneSets::PostLoop)); | ||
| } | ||
| } | ||
|
|
||
| #[derive(Component)] | ||
| pub struct VideoPlayer(pub PbVideoPlayer); | ||
|
|
||
| impl From<PbVideoPlayer> for VideoPlayer { | ||
| fn from(value: PbVideoPlayer) -> Self { | ||
| Self(value) | ||
| } | ||
| } | ||
|
|
||
| fn init_ffmpeg() { | ||
| ffmpeg_next::init().unwrap(); | ||
| ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Error); | ||
| } | ||
|
|
||
| fn play_videos(mut images: ResMut<Assets<Image>>, mut q: Query<&mut VideoSink>) { | ||
| for mut sink in q.iter_mut() { | ||
| match sink.data_receiver.try_recv() { | ||
| Ok(VideoData::Info(VideoInfo { | ||
| width, | ||
| height, | ||
| rate, | ||
| length, | ||
| })) => { | ||
| debug!("resize"); | ||
| images.get_mut(&sink.image).unwrap().resize(Extent3d { | ||
| width, | ||
| height, | ||
| depth_or_array_layers: 1, | ||
| }); | ||
| sink.length = Some(length); | ||
| sink.rate = Some(rate); | ||
| } | ||
| Ok(VideoData::Frame(frame, time)) => { | ||
| debug!("set frame on {:?}", sink.image); | ||
| images | ||
| .get_mut(&sink.image) | ||
| .unwrap() | ||
| .data | ||
| .copy_from_slice(frame.data(0)); | ||
| sink.current_time = time; | ||
| } | ||
| Err(_) => (), | ||
| } | ||
| } | ||
| } | ||
|
|
||
| pub fn update_video_players( | ||
| mut commands: Commands, | ||
| video_players: Query<(Entity, &VideoPlayer, Option<&VideoSink>), Changed<VideoPlayer>>, | ||
| mut images: ResMut<Assets<Image>>, | ||
| ) { | ||
| for (ent, player, maybe_sink) in video_players.iter() { | ||
| if maybe_sink.map(|sink| &sink.source) != Some(&player.0.src) { | ||
| let mut image = Image::new_fill( | ||
| bevy::render::render_resource::Extent3d { | ||
| width: 8, | ||
| height: 8, | ||
| depth_or_array_layers: 1, | ||
| }, | ||
| TextureDimension::D2, | ||
| &Color::PINK.as_rgba_u32().to_le_bytes(), | ||
| TextureFormat::Rgba8UnormSrgb, | ||
| ); | ||
| image.texture_descriptor.usage = | ||
| TextureUsages::COPY_DST | TextureUsages::TEXTURE_BINDING; | ||
| let image_handle = images.add(image); | ||
| let video_sink = VideoSink::new( | ||
| player.0.src.clone(), | ||
| image_handle, | ||
| player.0.playing.unwrap_or(true), | ||
| player.0.r#loop.unwrap_or(false), | ||
| ); | ||
| let video_output = VideoTextureOutput(video_sink.image.clone()); | ||
| commands.entity(ent).try_insert((video_sink, video_output)); | ||
| } else { | ||
| let sink = maybe_sink.as_ref().unwrap(); | ||
| if player.0.playing.unwrap_or(true) { | ||
| let _ = sink.command_sender.blocking_send(VideoCommand::Play); | ||
| } else { | ||
| let _ = sink.command_sender.blocking_send(VideoCommand::Pause); | ||
| } | ||
| let _ = sink | ||
| .command_sender | ||
| .blocking_send(VideoCommand::Repeat(player.0.r#loop.unwrap_or(false))); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.