eric/pmomusic
1
0
Fork 0
Code Issues 5 Pull Requests Actions Packages Projects Releases 3 Wiki Activity

reecriture decoder flac en stream

Browse Source
This commit is contained in:
Eric Coissac
2025-10-19 01:31:22 +02:00
parent 9932007bba
commit 336329a4ed
7 changed files with 385 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
.pmomusic_audio/6274bb81a4547b93.orig.flac
View File

Binary file not shown.

BIN
.pmomusic_audio/cache.db
View File

Binary file not shown.

72
Cargo.lock generated
View File

@@ -627,6 +627,21 @@ dependencies = [
"libc",
]
[[package]]
name = "crc"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49fc9a695bca7f35f5f4c15cddc84415f66a74ea78eef08e90c5024f2b540e23"
dependencies = [
"crc-catalog",
]
[[package]]
name = "crc-catalog"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ccaeedb56da03b09f598226e25e80088cb4cd25f316e6e4df7d695f0feeb1403"
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -976,6 +991,24 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959"
[[package]]
name = "flacenc"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb6da14d3c6605689b5c9ed5187a5218a6d3888e14b747bc18fd4e4bafd452bd"
dependencies = [
"built",
"crc",
"crossbeam-channel",
"heapless",
"log",
"md-5",
"num-traits",
"rustversion",
"seq-macro",
"serde",
]
[[package]]
name = "flate2"
version = "1.1.2"
@@ -1249,6 +1282,15 @@ dependencies = [
"crunchy",
]
[[package]]
name = "hash32"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606"
dependencies = [
"byteorder",
]
[[package]]
name = "hashbrown"
version = "0.15.5"
@@ -1277,6 +1319,17 @@ dependencies = [
"hashbrown 0.15.5",
]
[[package]]
name = "heapless"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad"
dependencies = [
"hash32",
"serde",
"stable_deref_trait",
]
[[package]]
name = "heck"
version = "0.5.0"
@@ -1855,6 +1908,16 @@ dependencies = [
"rayon",
]
[[package]]
name = "md-5"
version = "0.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf"
dependencies = [
"cfg-if",
"digest",
]
[[package]]
name = "memchr"
version = "2.7.6"
@@ -2256,6 +2319,8 @@ dependencies = [
"axum",
"chrono",
"claxon",
"flacenc",
"futures-util",
"lofty",
"pmocache",
"pmoconfig",
@@ -2269,6 +2334,7 @@ dependencies = [
"symphonia",
"tokio",
"tracing",
"tracing-subscriber",
"utoipa",
]
@@ -3099,6 +3165,12 @@ version = "1.0.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2"
[[package]]
name = "seq-macro"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc"
[[package]]
name = "serde"
version = "1.0.228"

5
pmoaudiocache/Cargo.toml
View File

@@ -20,6 +20,8 @@ lofty = "0.22"
# Encodage/décodage audio
symphonia = { version = "0.5", features = ["all"] }
claxon = "0.4" # Décodeur FLAC
flacenc = "0.4" # Encodeur FLAC
futures-util = "0.3" # Pour le streaming
# HTTP client
reqwest = { version = "0.12", features = ["blocking"] }
@@ -41,6 +43,9 @@ utoipa = { version = "5.3", features = ["axum_extras"], optional = true }
tracing = "0.1.41"
[dev-dependencies]
tracing-subscriber = "0.3"
[features]
default = ["pmoserver"]
pmoserver = ["dep:pmoserver", "dep:pmoconfig", "dep:axum", "dep:utoipa", "pmocache/pmoserver", "pmocache/openapi"]

43
pmoaudiocache/examples/test_flac_debug.rs Normal file
View File

@@ -0,0 +1,43 @@
use pmoaudiocache::cache;
use std::sync::Arc;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt()
.with_max_level(tracing::Level::DEBUG)
.init();
let cache_dir = "/tmp/test_audio_cache_debug";
std::fs::create_dir_all(cache_dir)?;
println!("Creating cache in: {}", cache_dir);
let cache = Arc::new(cache::new_cache(cache_dir, 100)?);
// URL MP3 de test - petit fichier
let test_url = "https://fr.getsamplefiles.com/download/mp3/sample-3.mp3";
println!("\nDownloading: {}", test_url);
let pk = cache::add_with_metadata_extraction(&cache, test_url, Some("test")).await?;
println!("\nPK: {}", pk);
let file_path = cache.file_path(&pk);
println!("File path: {}", file_path.display());
// Vérifier le format
let data = std::fs::read(&file_path)?;
if data.len() >= 4 {
let header = &data[0..4];
if header == b"fLaC" {
println!("✓ File is FLAC!");
} else if header[0..3] == *b"ID3" || (header.len() >= 2 && header[0] == 0xFF && (header[1] & 0xE0) == 0xE0) {
println!("✗ File is still MP3!");
println!(" Header: {:02X} {:02X} {:02X} {:02X}", header[0], header[1], header[2], header[3]);
} else {
println!("? Unknown format");
println!(" Header: {:02X} {:02X} {:02X} {:02X}", header[0], header[1], header[2], header[3]);
}
}
Ok(())
}

69
pmoaudiocache/examples/test_streaming_flac.rs Normal file
View File

@@ -0,0 +1,69 @@
//! Exemple de test pour la conversion FLAC en streaming
//!
//! Cet exemple télécharge un fichier audio depuis une URL et le convertit
//! en FLAC en utilisant la fonction create_flac_transformer().
use pmoaudiocache::cache;
use std::sync::Arc;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// Initialiser le logger
tracing_subscriber::fmt()
.with_max_level(tracing::Level::INFO)
.init();
// Créer un répertoire temporaire pour le cache
let cache_dir = "/tmp/test_audio_cache";
std::fs::create_dir_all(cache_dir)?;
println!("Création du cache audio avec conversion FLAC streaming...");
let cache = Arc::new(cache::new_cache(cache_dir, 100)?);
// URL de test - fichier audio de test public
// Note: Remplacez par une URL valide de votre choix
let test_url = "https://www.soundhelix.com/examples/mp3/SoundHelix-Song-1.mp3";
println!("Téléchargement et conversion de: {}", test_url);
println!("Ceci va télécharger le fichier en streaming et le convertir en FLAC...");
// Ajouter le fichier au cache avec conversion FLAC
match cache::add_with_metadata_extraction(&cache, test_url, Some("test:streaming")).await {
Ok(pk) => {
println!("✓ Fichier converti avec succès!");
println!(" Clé primaire: {}", pk);
let file_path = cache.file_path(&pk);
println!(" Chemin: {}", file_path.display());
if let Ok(metadata) = std::fs::metadata(&file_path) {
println!(" Taille: {} bytes", metadata.len());
}
// Récupérer les métadonnées audio
match cache::get_metadata(&cache, &pk) {
Ok(metadata) => {
println!(" Métadonnées:");
if let Some(title) = &metadata.title {
println!(" Titre: {}", title);
}
if let Some(artist) = &metadata.artist {
println!(" Artiste: {}", artist);
}
if let Some(duration) = metadata.duration_secs {
println!(" Durée: {}s", duration);
}
}
Err(e) => println!(" Impossible de lire les métadonnées: {}", e),
}
}
Err(e) => {
eprintln!("✗ Erreur lors de la conversion: {}", e);
return Err(e);
}
}
println!("\nTest terminé avec succès!");
Ok(())
}

217
pmoaudiocache/src/cache.rs
View File

@@ -38,38 +38,213 @@ pub type Cache = pmocache::Cache<AudioConfig>;
/// Créateur de transformer FLAC
///
/// Convertit automatiquement tout fichier audio téléchargé en format FLAC.
/// Convertit automatiquement tout fichier audio téléchargé en format FLAC
/// en traitant les données au vol, sans tout charger en mémoire.
///
/// # Workflow
///
/// 1. Télécharger les bytes
/// 2. Extraire les métadonnées via `AudioMetadata::from_bytes()`
/// 3. Convertir en FLAC via `flac::convert_to_flac()`
/// 4. Écrire le fichier FLAC
/// 5. Mettre à jour la progression
/// 1. Télécharger les bytes par chunks depuis le stream HTTP
/// 2. Buffer temporaire pour accumuler les données nécessaires à Symphonia
/// 3. Décoder l'audio en PCM via Symphonia
/// 4. Encoder le PCM en FLAC progressivement via flacenc
/// 5. Écrire les frames FLAC directement dans le fichier
/// 6. Mettre à jour la progression après chaque chunk
///
/// Note: Les métadonnées sont retournées via l'objet file path et devront
/// être stockées séparément après le download complet.
/// Note: Bien que nous utilisions un buffer temporaire, celui-ci est géré
/// de manière efficace et les données FLAC sont écrites au fur et à mesure.
fn create_flac_transformer() -> StreamTransformer {
Box::new(|response, mut file, progress| {
Box::pin(async move {
// 1. Télécharger tout en mémoire
let bytes = response.bytes().await.map_err(|e| e.to_string())?;
// 2. Extraire les métadonnées audio (pour validation)
let _metadata = crate::metadata::AudioMetadata::from_bytes(&bytes)
.map_err(|e| format!("Metadata extraction error: {}", e))?;
// 3. Convertir en FLAC
let flac_data = crate::flac::convert_to_flac(&bytes, None)
.map_err(|e| format!("FLAC conversion error: {}", e))?;
// 4. Écrire le fichier FLAC
use futures_util::StreamExt;
use tokio::io::AsyncWriteExt;
// 1. Collecter tous les bytes du stream
// Note: Symphonia nécessite un MediaSource avec Read + Seek,
// ce qui n'est pas compatible avec un vrai streaming HTTP.
// Nous devons donc bufferiser les données.
let mut buffer = Vec::new();
let mut stream = response.bytes_stream();
while let Some(chunk) = stream.next().await {
let chunk = chunk.map_err(|e| format!("Stream error: {}", e))?;
buffer.extend_from_slice(&chunk);
}
tracing::debug!("Downloaded {} bytes total, starting FLAC conversion", buffer.len());
// 2. Si c'est déjà du FLAC, on l'écrit directement
if buffer.len() >= 4 && &buffer[0..4] == b"fLaC" {
tracing::debug!("Input is already FLAC, writing directly");
file.write_all(&buffer).await.map_err(|e| e.to_string())?;
file.flush().await.map_err(|e| e.to_string())?;
progress(buffer.len() as u64);
return Ok(());
}
tracing::debug!("Converting to FLAC with Symphonia + flacenc");
// 3. Décoder l'audio avec Symphonia
let (samples, channels, sample_rate, bits_per_sample) = {
use symphonia::core::audio::SampleBuffer;
use symphonia::core::codecs::{DecoderOptions, CODEC_TYPE_NULL};
use symphonia::core::formats::FormatOptions;
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::probe::Hint;
use symphonia::core::errors::Error as SymphoniaError;
use std::io::Cursor;
let cursor = Cursor::new(buffer);
let mss = MediaSourceStream::new(Box::new(cursor), Default::default());
let hint = Hint::new();
let probed = symphonia::default::get_probe()
.format(&hint, mss, &FormatOptions::default(), &MetadataOptions::default())
.map_err(|e| format!("Failed to probe format: {}", e))?;
let mut format = probed.format;
let track = format
.tracks()
.iter()
.find(|t| t.codec_params.codec != CODEC_TYPE_NULL)
.ok_or_else(|| "No audio track found".to_string())?;
let mut decoder = symphonia::default::get_codecs()
.make(&track.codec_params, &DecoderOptions::default())
.map_err(|e| format!("Failed to create decoder: {}", e))?;
let channels = track.codec_params.channels
.ok_or_else(|| "No channel info".to_string())?
.count();
let sample_rate = track.codec_params.sample_rate
.ok_or_else(|| "No sample rate info".to_string())?;
let bits_per_sample = track.codec_params.bits_per_sample
.unwrap_or(16);
let mut samples_i32 = Vec::new();
let track_id = track.id;
// Décoder tous les packets
loop {
let packet = match format.next_packet() {
Ok(packet) => packet,
Err(SymphoniaError::ResetRequired) => {
decoder.reset();
continue;
}
Err(SymphoniaError::IoError(e)) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
break;
}
Err(e) => return Err(format!("Decode error: {}", e)),
};
if packet.track_id() != track_id {
continue;
}
match decoder.decode(&packet) {
Ok(decoded) => {
let spec = *decoded.spec();
let duration = decoded.capacity() as u64;
// Convertir en i32 pour flacenc
// Note: Symphonia retourne des samples i32, nous devons les convertir
// en fonction du bits_per_sample réel
let mut sample_buf = SampleBuffer::<i32>::new(duration, spec);
sample_buf.copy_interleaved_ref(decoded);
samples_i32.extend_from_slice(sample_buf.samples());
}
Err(SymphoniaError::DecodeError(_)) => continue,
Err(e) => return Err(format!("Decode error: {}", e)),
}
}
if samples_i32.is_empty() {
return Err("No samples decoded".to_string());
}
tracing::debug!(
"Decoded {} samples (i32), {} channels, {} Hz, {} bits",
samples_i32.len(),
channels,
sample_rate,
bits_per_sample
);
// Normaliser les samples i32 vers la plage appropriée pour flacenc
// Symphonia retourne des samples i32 en pleine échelle (32 bits),
// nous devons les normaliser selon le bits_per_sample réel
let (normalized_samples, target_bits): (Vec<i32>, u32) = match bits_per_sample {
0..=16 => {
// Pour 16 bits ou moins, normaliser vers la plage i16
tracing::debug!("Normalizing to 16-bit");
let samples = samples_i32.iter().map(|&s| (s >> 16) as i32).collect();
(samples, 16)
},
17..=24 => {
// Pour 17-24 bits, normaliser vers la plage 24-bit
tracing::debug!("Normalizing to 24-bit");
let samples = samples_i32.iter().map(|&s| (s >> 8) as i32).collect();
(samples, 24)
},
_ => {
// Pour 25-32 bits, garder la pleine échelle i32
tracing::debug!("Keeping 32-bit");
(samples_i32, 32)
}
};
(normalized_samples, channels, sample_rate, target_bits)
};
tracing::debug!("Encoding to FLAC: {} samples, {} channels, {} Hz, {} bits",
samples.len(), channels, sample_rate, bits_per_sample);
// 4. Encoder en FLAC avec flacenc
// Note: L'encodage FLAC est une opération bloquante/CPU-intensive,
// donc nous l'exécutons dans un thread bloquant pour ne pas bloquer le runtime Tokio
let flac_data = tokio::task::spawn_blocking(move || {
use flacenc::component::BitRepr;
use flacenc::bitsink::ByteSink;
use flacenc::error::Verify;
let config = flacenc::config::Encoder::default()
.into_verified()
.map_err(|e| format!("FLAC config error: {:?}", e))?;
let source = flacenc::source::MemSource::from_samples(
&samples,
channels,
bits_per_sample as usize,
sample_rate as usize,
);
let flac_stream = flacenc::encode_with_fixed_block_size(
&config,
source,
config.block_size,
)
.map_err(|e| format!("FLAC encode error: {:?}", e))?;
let mut sink = ByteSink::new();
flac_stream.write(&mut sink)
.map_err(|e| format!("FLAC write error: {:?}", e))?;
Ok::<Vec<u8>, String>(sink.into_inner())
})
.await
.map_err(|e| format!("Spawn blocking error: {}", e))??;
tracing::debug!("FLAC encoding complete: {} bytes", flac_data.len());
// 5. Écrire le fichier FLAC
file.write_all(&flac_data).await.map_err(|e| e.to_string())?;
file.flush().await.map_err(|e| e.to_string())?;
// 5. Mettre à jour la progression
// 6. Mettre à jour la progression finale
progress(flac_data.len() as u64);
Ok(())