From 47cf586116c09f853780bf934deb7a09afd698cb Mon Sep 17 00:00:00 2001 From: CodeSteak Date: Mon, 3 Jun 2019 21:21:52 +0200 Subject: [PATCH] [dirty] Implement Background Preview Generation --- server/texture-sync-server/Cargo.toml | 3 +- server/texture-sync-server/src/main.rs | 2 + .../src/model/texture_format.rs | 2 +- .../src/persistency/image_convert/mod.rs | 4 +- .../src/persistency/mod.rs | 88 +++++++++++++++++-- 5 files changed, 86 insertions(+), 13 deletions(-) diff --git a/server/texture-sync-server/Cargo.toml b/server/texture-sync-server/Cargo.toml index 7b9dcec..901e2d7 100644 --- a/server/texture-sync-server/Cargo.toml +++ b/server/texture-sync-server/Cargo.toml @@ -5,8 +5,9 @@ authors = ["CodeSteak "] edition = "2018" [dependencies] -image = "0.21.1" +image = { version = "0.21.1", default-features = false, features = ["jpeg", "png_codec" ] } serde = { version = "1.0.90", features = ["derive"] } serde_json = "1.0.39" sha2 = "0.8.0" lovecraft = "0.2.0" +num_cpus = "1.0" \ No newline at end of file diff --git a/server/texture-sync-server/src/main.rs b/server/texture-sync-server/src/main.rs index 7507123..f6711bc 100644 --- a/server/texture-sync-server/src/main.rs +++ b/server/texture-sync-server/src/main.rs @@ -8,6 +8,8 @@ extern crate lovecraft; extern crate sha2; +extern crate num_cpus; + pub mod model; pub mod persistency; pub mod protocol; diff --git a/server/texture-sync-server/src/model/texture_format.rs b/server/texture-sync-server/src/model/texture_format.rs index 2579471..d9cbb11 100644 --- a/server/texture-sync-server/src/model/texture_format.rs +++ b/server/texture-sync-server/src/model/texture_format.rs @@ -3,7 +3,7 @@ use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)] +#[derive(Copy, Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)] pub enum TextureFormat { #[serde(rename = "png")] PNG, diff --git a/server/texture-sync-server/src/persistency/image_convert/mod.rs b/server/texture-sync-server/src/persistency/image_convert/mod.rs index 2c609de..c608f1a 100644 --- a/server/texture-sync-server/src/persistency/image_convert/mod.rs +++ b/server/texture-sync-server/src/persistency/image_convert/mod.rs @@ -10,12 +10,12 @@ pub fn generate_preview(input: &[u8], format: TextureFormat) -> ImageResult = Vec::new(); match format { - TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(95))?, + TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(85))?, TextureFormat::PNG => img.write_to(&mut out, ImageOutputFormat::PNG)?, } diff --git a/server/texture-sync-server/src/persistency/mod.rs b/server/texture-sync-server/src/persistency/mod.rs index 950a9a1..573cbb8 100644 --- a/server/texture-sync-server/src/persistency/mod.rs +++ b/server/texture-sync-server/src/persistency/mod.rs @@ -3,10 +3,13 @@ use crate::model::*; use std::collections::*; use std::io; use std::path::{Path, PathBuf}; +use std::sync::RwLock; +use std::sync::{Arc, Mutex}; mod image_convert; mod metadata_file; +type PreviewCache = Arc>>>; pub struct DataStore { //data_path: PathBuf, base_dir: PathBuf, @@ -16,7 +19,7 @@ pub struct DataStore { id_index: HashMap, name_index: HashMap, - preview_cache: HashMap<(TextureFormat, Sha256), Vec>, + preview_cache: PreviewCache, } impl DataStore { @@ -38,6 +41,18 @@ impl DataStore { path } + fn light_clone(&self) -> DataStore { + DataStore { + base_dir: self.base_dir.clone(), + + textures: Default::default(), + id_index: Default::default(), + name_index: Default::default(), + + preview_cache: Default::default(), + } + } + pub fn new(base_dir: &Path) -> io::Result { let mut store = DataStore { base_dir: base_dir.into(), @@ -51,7 +66,11 @@ impl DataStore { let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?; + let mut preview_cache_tasks: Vec<(TextureFormat, Sha256)> = vec![]; + for texture in metadata_file.textures.iter() { + preview_cache_tasks.push((TextureFormat::JPEG, texture.texture_hash.clone())); + match store.insert(texture.clone()) { true => (), false => { @@ -63,6 +82,42 @@ impl DataStore { store.garbage_collect()?; store.flush_metadata()?; + // Background Preview Generation + { + let number_of_tasks = preview_cache_tasks.len(); + let preview_cache_tasks = Arc::new(Mutex::new(preview_cache_tasks)); + + println!("Generating Previews"); + for _ in 0..(num_cpus::get() * 2) { + let cache = store.preview_cache.clone(); + let light_copy = store.light_clone(); + let preview_cache_tasks = preview_cache_tasks.clone(); + + std::thread::spawn(move || { + while let (Some((format, hash)), n) = { + let mut tasks = preview_cache_tasks.lock().unwrap(); + (tasks.pop(), tasks.len()) + } { + if n % 100 == 0 || n == number_of_tasks - 1 { + println!( + "[Background] {:5}/{:5} Previews left to generate.", + n, number_of_tasks + ) + } + + match light_copy.calculate_texture_preview(&hash, format) { + Ok(preview) => { + cache.write().unwrap().insert((format, hash), preview); + } + Err(e) => { + println!("[Background] Error, Skipped Generated Preview '{}' {:?} \n\t {:?}", hash, format, e); + } + } + } + }); + } + } + Ok(store) } @@ -151,6 +206,23 @@ impl DataStore { f.store(self.index_file_path().as_path()) } + pub fn calculate_texture_preview( + &self, + hash: &Sha256, + desired_format: TextureFormat, + ) -> io::Result> { + let original = self.read_texture_file_by_hash(&hash)?; + + let mut preview = + image_convert::generate_preview(&original[..], desired_format).map_err(|_e| { + io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.") + })?; + + preview.shrink_to_fit(); + + Ok(preview) + } + pub fn get_texture_preview( &mut self, hash: &Sha256, @@ -158,18 +230,16 @@ impl DataStore { ) -> io::Result> { let key = (desired_format.clone(), hash.clone()); - if let Some(preview) = self.preview_cache.get(&key) { + if let Some(preview) = self.preview_cache.read().unwrap().get(&key) { return Ok(preview.clone()); } - let original = self.read_texture_file_by_hash(&hash)?; + let preview = self.calculate_texture_preview(hash, desired_format)?; - let preview = - image_convert::generate_preview(&original[..], desired_format).map_err(|_e| { - io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.") - })?; - - self.preview_cache.insert(key, preview.clone()); + self.preview_cache + .write() + .unwrap() + .insert(key, preview.clone()); Ok(preview) }