Skip to content

Commit

Permalink
Audio/Video (#39)
Browse files Browse the repository at this point in the history
- add static audio with spatial params via kira
- add video playback via ffmpeg
- remove macos aarch64 build due to ffmpeg issues
- minor fixes
  - sphere radius -> 0.5
  - camera mode area -> allowed on root
  - hopefully fix avatar race condition on startup
  • Loading branch information
robtfm authored Jul 6, 2023
1 parent a63328c commit 6735a0e
Show file tree
Hide file tree
Showing 26 changed files with 629 additions and 30 deletions.
23 changes: 23 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ jobs:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install alsa and udev
run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev
- name: install ffmpeg deps
run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
- uses: actions/checkout@v2
- uses: actions-rs/toolchain@v1
with:
Expand Down Expand Up @@ -43,6 +45,23 @@ jobs:
- name: Install alsa and udev
run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev
if: runner.os == 'linux'
- name: install ffmpeg deps (linux)
run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
if: runner.os == 'linux'
- name: install ffmpeg deps (macOs)
if: runner.os == 'macos'
run: brew install ffmpeg pkg-config
- name: install ffmpeg deps (windows)
if: runner.os == 'windows'
run: |
$VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
Invoke-WebRequest "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-full-shared.7z" -OutFile ffmpeg-release-full-shared.7z
7z x ffmpeg-release-full-shared.7z
mkdir ffmpeg
mv ffmpeg-*/* ffmpeg/
Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
- uses: actions-rs/cargo@v1
with:
command: test
Expand All @@ -65,6 +84,8 @@ jobs:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install alsa and udev
run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev
- name: install ffmpeg deps (linux)
run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
- uses: actions-rs/cargo@v1
with:
command: fmt
Expand All @@ -87,6 +108,8 @@ jobs:
repo-token: ${{ secrets.GITHUB_TOKEN }}
- name: Install alsa and udev
run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev
- name: install ffmpeg deps (linux)
run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
- uses: actions-rs/cargo@v1
with:
command: clippy
Expand Down
27 changes: 23 additions & 4 deletions .github/workflows/package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,10 @@ jobs:
os: macos-latest
target: x86_64-apple-darwin

- build_name: macos-aarch64
os: macos-latest
target: aarch64-apple-darwin
# ffmpeg does not cross-build to aarch64 in CI, so aarch64 releases must be manually added
# - build_name: macos-aarch64
# os: macos-latest
# target: aarch64-apple-darwin

- build_name: windows-x86_64
os: windows-latest
Expand Down Expand Up @@ -85,7 +86,25 @@ jobs:
- name: Install alsa and udev
if: runner.os == 'linux'
run: sudo apt-get update; sudo apt-get install --no-install-recommends libasound2-dev libudev-dev


- name: install ffmpeg deps (linux)
run: sudo apt install -y --no-install-recommends clang curl pkg-config libavcodec-dev libavformat-dev libavutil-dev libavfilter-dev libavdevice-dev
if: runner.os == 'linux'
- name: install ffmpeg deps (macOs)
if: runner.os == 'macos'
run: brew install ffmpeg pkg-config
- name: install ffmpeg deps (windows)
if: runner.os == 'windows'
run: |
$VCINSTALLDIR = $(& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" -latest -property installationPath)
Add-Content $env:GITHUB_ENV "LIBCLANG_PATH=${VCINSTALLDIR}\VC\Tools\LLVM\x64\bin`n"
Invoke-WebRequest "https://www.gyan.dev/ffmpeg/builds/ffmpeg-release-full-shared.7z" -OutFile ffmpeg-release-full-shared.7z
7z x ffmpeg-release-full-shared.7z
mkdir ffmpeg
mv ffmpeg-*/* ffmpeg/
Add-Content $env:GITHUB_ENV "FFMPEG_DIR=${pwd}\ffmpeg`n"
Add-Content $env:GITHUB_PATH "${pwd}\ffmpeg\bin`n"
- name: Cargo build
run: cargo build --release ${{ matrix.target && '--target' }} ${{ matrix.target }}
env:
Expand Down
2 changes: 2 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ visuals = { path="crates/visuals" }
ui_core = { path="crates/ui_core" }
scene_runner = { path="crates/scene_runner" }
console = { path="crates/console" }
av = { path="crates/av" }

bevy = "0.10"
bevy_console = "0.7.0"
Expand All @@ -39,4 +40,5 @@ prost-build = "0.11.8"

[patch.crates-io]
bevy = { git="https://github.com/robtfm/bevy", branch="0.10-hotfix" }
# bevy = { path = "../bevy" }
bevy_mod_billboard = { git="https://github.com/robtfm/bevy_mod_billboard", branch="lock_all" }
19 changes: 19 additions & 0 deletions crates/av/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
[package]
name = "av"
version = "0.1.0"
edition = "2021"

[lib]

[dependencies]
common = { path = "../common" }
dcl = { path="../dcl" }
dcl_component = { path="../dcl_component" }
ipfs = { path="../ipfs" }
scene_runner = { path="../scene_runner" }

bevy = "0.10"
bevy_kira_audio = { version= "0.15", features=["flac", "mp3", "ogg", "wav"] }
ffmpeg-next = "6.0.0"
tokio = { version = "1.29.1", features = ["sync"] }
anyhow = "1.0.70"
99 changes: 99 additions & 0 deletions crates/av/src/audio_source.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,99 @@
use std::time::Duration;

use bevy::prelude::*;
use bevy_kira_audio::{
prelude::{AudioEmitter, AudioReceiver},
AudioControl, AudioInstance, AudioTween,
};
use common::{structs::PrimaryCameraRes, util::TryInsertEx};
use dcl_component::proto_components::sdk::components::PbAudioSource;
use ipfs::IpfsLoaderExt;
use scene_runner::{renderer_context::RendererSceneContext, SceneEntity};

#[derive(Component, Debug)]
pub struct AudioSource(PbAudioSource);

impl From<PbAudioSource> for AudioSource {
fn from(value: PbAudioSource) -> Self {
Self(value)
}
}

pub(crate) fn setup_audio(mut commands: Commands, camera: Res<PrimaryCameraRes>) {
commands.entity(camera.0).try_insert(AudioReceiver);
}

#[allow(clippy::type_complexity)]
pub(crate) fn update_audio(
mut commands: Commands,
mut query: Query<
(
Entity,
&SceneEntity,
&AudioSource,
Option<&Handle<bevy_kira_audio::AudioSource>>,
Option<&mut AudioEmitter>,
),
Changed<AudioSource>,
>,
scenes: Query<&RendererSceneContext>,
audio: Res<bevy_kira_audio::Audio>,
asset_server: Res<AssetServer>,
mut audio_instances: ResMut<Assets<AudioInstance>>,
) {
for (ent, scene_ent, audio_source, maybe_source, maybe_emitter) in query.iter_mut() {
// preload clips
let h_audio = match maybe_source {
Some(instance) => instance.clone_weak(),
None => {
let Ok(scene) = scenes.get(scene_ent.root) else {
warn!("failed to load audio source scene");
continue;
};

let Ok(handle) = asset_server.load_content_file(&audio_source.0.audio_clip_url, &scene.hash) else {
warn!("failed to load content file");
continue;
};

let h_audio = handle.clone_weak();
commands.entity(ent).try_insert(handle);
h_audio
}
};

if audio_source.0.playing() {
// stop previous - is this right?
if let Some(emitter) = maybe_emitter {
for h_instance in emitter.instances.iter() {
if let Some(instance) = audio_instances.get_mut(h_instance) {
instance.stop(AudioTween::linear(Duration::ZERO));
}
}
}

let mut instance = &mut audio.play(h_audio);
if audio_source.0.r#loop() {
instance = instance.looped();
}

if let Some(volume) = audio_source.0.volume {
instance = instance
.with_volume(bevy_kira_audio::prelude::Volume::Amplitude(volume as f64));
}

let instance = instance.handle();
commands.entity(ent).try_insert(AudioEmitter {
instances: vec![instance],
});
} else if let Some(mut emitter) = maybe_emitter {
// stop running
for h_instance in emitter.instances.iter() {
if let Some(instance) = audio_instances.get_mut(h_instance) {
instance.stop(AudioTween::linear(Duration::ZERO));
}
}
emitter.instances.clear();
}
}
}
28 changes: 28 additions & 0 deletions crates/av/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
pub mod audio_source;
pub mod video_player;
pub mod video_thread;

use audio_source::{setup_audio, update_audio};
use bevy::prelude::*;
use bevy_kira_audio::prelude::SpacialAudio;
use common::sets::{SceneSets, SetupSets};
use dcl::interface::ComponentPosition;
use dcl_component::{proto_components::sdk::components::PbAudioSource, SceneComponentId};
use scene_runner::update_world::AddCrdtInterfaceExt;
use video_player::VideoPlayerPlugin;

pub struct AudioPlugin;

impl Plugin for AudioPlugin {
fn build(&self, app: &mut App) {
app.add_plugin(bevy_kira_audio::AudioPlugin);
app.add_plugin(VideoPlayerPlugin);
app.add_crdt_lww_component::<PbAudioSource, audio_source::AudioSource>(
SceneComponentId::AUDIO_SOURCE,
ComponentPosition::EntityOnly,
);
app.add_system(update_audio.in_set(SceneSets::PostLoop));
app.insert_resource(SpacialAudio { max_distance: 25. });
app.add_startup_system(setup_audio.in_set(SetupSets::Main));
}
}
114 changes: 114 additions & 0 deletions crates/av/src/video_player.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
// TODO: rate, audio channels, update position(?)

use bevy::{
prelude::*,
render::render_resource::{Extent3d, TextureDimension, TextureFormat, TextureUsages},
};
use common::{sets::SceneSets, util::TryInsertEx};
use dcl::interface::ComponentPosition;
use dcl_component::{proto_components::sdk::components::PbVideoPlayer, SceneComponentId};
use scene_runner::update_world::{material::VideoTextureOutput, AddCrdtInterfaceExt};

use crate::video_thread::{VideoCommand, VideoData, VideoInfo, VideoSink};

pub struct VideoPlayerPlugin;

impl Plugin for VideoPlayerPlugin {
fn build(&self, app: &mut App) {
app.add_crdt_lww_component::<PbVideoPlayer, VideoPlayer>(
SceneComponentId::VIDEO_PLAYER,
ComponentPosition::EntityOnly,
);
app.add_startup_system(init_ffmpeg);
app.add_system(play_videos);
app.add_system(update_video_players.in_set(SceneSets::PostLoop));
}
}

#[derive(Component)]
pub struct VideoPlayer(pub PbVideoPlayer);

impl From<PbVideoPlayer> for VideoPlayer {
fn from(value: PbVideoPlayer) -> Self {
Self(value)
}
}

fn init_ffmpeg() {
ffmpeg_next::init().unwrap();
ffmpeg_next::log::set_level(ffmpeg_next::log::Level::Error);
}

fn play_videos(mut images: ResMut<Assets<Image>>, mut q: Query<&mut VideoSink>) {
for mut sink in q.iter_mut() {
match sink.data_receiver.try_recv() {
Ok(VideoData::Info(VideoInfo {
width,
height,
rate,
length,
})) => {
debug!("resize");
images.get_mut(&sink.image).unwrap().resize(Extent3d {
width,
height,
depth_or_array_layers: 1,
});
sink.length = Some(length);
sink.rate = Some(rate);
}
Ok(VideoData::Frame(frame, time)) => {
debug!("set frame on {:?}", sink.image);
images
.get_mut(&sink.image)
.unwrap()
.data
.copy_from_slice(frame.data(0));
sink.current_time = time;
}
Err(_) => (),
}
}
}

pub fn update_video_players(
mut commands: Commands,
video_players: Query<(Entity, &VideoPlayer, Option<&VideoSink>), Changed<VideoPlayer>>,
mut images: ResMut<Assets<Image>>,
) {
for (ent, player, maybe_sink) in video_players.iter() {
if maybe_sink.map(|sink| &sink.source) != Some(&player.0.src) {
let mut image = Image::new_fill(
bevy::render::render_resource::Extent3d {
width: 8,
height: 8,
depth_or_array_layers: 1,
},
TextureDimension::D2,
&Color::PINK.as_rgba_u32().to_le_bytes(),
TextureFormat::Rgba8UnormSrgb,
);
image.texture_descriptor.usage =
TextureUsages::COPY_DST | TextureUsages::TEXTURE_BINDING;
let image_handle = images.add(image);
let video_sink = VideoSink::new(
player.0.src.clone(),
image_handle,
player.0.playing.unwrap_or(true),
player.0.r#loop.unwrap_or(false),
);
let video_output = VideoTextureOutput(video_sink.image.clone());
commands.entity(ent).try_insert((video_sink, video_output));
} else {
let sink = maybe_sink.as_ref().unwrap();
if player.0.playing.unwrap_or(true) {
let _ = sink.command_sender.blocking_send(VideoCommand::Play);
} else {
let _ = sink.command_sender.blocking_send(VideoCommand::Pause);
}
let _ = sink
.command_sender
.blocking_send(VideoCommand::Repeat(player.0.r#loop.unwrap_or(false)));
}
}
}
Loading

0 comments on commit 6735a0e

Please sign in to comment.