localhorst
/
TextureSync
3
1
Fork 0
Code Issues 2 Pull Requests Releases 3 Wiki Activity

[dirty] Implement Background Preview Generation

This commit is contained in:
CodeSteak 2019-06-03 21:21:52 +02:00
parent b2ba9a9ba4
commit 47cf586116
5 changed files with 86 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server/texture-sync-server/Cargo.toml
View File

@ -5,8 +5,9 @@ authors = ["CodeSteak <codesteak@shellf.art>"]
edition = "2018"
[dependencies]
image = "0.21.1"
image = { version = "0.21.1", default-features = false, features = ["jpeg", "png_codec" ] }
serde = { version = "1.0.90", features = ["derive"] }
serde_json = "1.0.39"
sha2 = "0.8.0"
lovecraft = "0.2.0"
num_cpus = "1.0"

2
server/texture-sync-server/src/main.rs
View File

@ -8,6 +8,8 @@ extern crate lovecraft;
extern crate sha2;
extern crate num_cpus;
pub mod model;
pub mod persistency;
pub mod protocol;

2
server/texture-sync-server/src/model/texture_format.rs
View File

@ -3,7 +3,7 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
#[derive(Copy, Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
pub enum TextureFormat {
#[serde(rename = "png")]
PNG,

4
server/texture-sync-server/src/persistency/image_convert/mod.rs
View File

@ -10,12 +10,12 @@ pub fn generate_preview(input: &[u8], format: TextureFormat) -> ImageResult<Vec<
// Yes, this guesses the format :D
// Also the resize function takes a maximum size and preservs the ratio.
let img =
::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Lanczos3);
::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Nearest);
let mut out: Vec<u8> = Vec::new();
match format {
TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(95))?,
TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(85))?,
TextureFormat::PNG => img.write_to(&mut out, ImageOutputFormat::PNG)?,
}

88
server/texture-sync-server/src/persistency/mod.rs
View File

@ -3,10 +3,13 @@ use crate::model::*;
use std::collections::*;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::RwLock;
use std::sync::{Arc, Mutex};
mod image_convert;
mod metadata_file;
type PreviewCache = Arc<RwLock<HashMap<(TextureFormat, Sha256), Vec<u8>>>>;
pub struct DataStore {
//data_path: PathBuf,
base_dir: PathBuf,
@ -16,7 +19,7 @@ pub struct DataStore {
id_index: HashMap<String, Texture>,
name_index: HashMap<String, Texture>,
preview_cache: HashMap<(TextureFormat, Sha256), Vec<u8>>,
preview_cache: PreviewCache,
}
impl DataStore {
@ -38,6 +41,18 @@ impl DataStore {
path
}
fn light_clone(&self) -> DataStore {
DataStore {
base_dir: self.base_dir.clone(),
textures: Default::default(),
id_index: Default::default(),
name_index: Default::default(),
preview_cache: Default::default(),
}
}
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
let mut store = DataStore {
base_dir: base_dir.into(),
@ -51,7 +66,11 @@ impl DataStore {
let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?;
let mut preview_cache_tasks: Vec<(TextureFormat, Sha256)> = vec![];
for texture in metadata_file.textures.iter() {
preview_cache_tasks.push((TextureFormat::JPEG, texture.texture_hash.clone()));
match store.insert(texture.clone()) {
true => (),
false => {
@ -63,6 +82,42 @@ impl DataStore {
store.garbage_collect()?;
store.flush_metadata()?;
// Background Preview Generation
{
let number_of_tasks = preview_cache_tasks.len();
let preview_cache_tasks = Arc::new(Mutex::new(preview_cache_tasks));
println!("Generating Previews");
for _ in 0..(num_cpus::get() * 2) {
let cache = store.preview_cache.clone();
let light_copy = store.light_clone();
let preview_cache_tasks = preview_cache_tasks.clone();
std::thread::spawn(move || {
while let (Some((format, hash)), n) = {
let mut tasks = preview_cache_tasks.lock().unwrap();
(tasks.pop(), tasks.len())
} {
if n % 100 == 0 || n == number_of_tasks - 1 {
println!(
"[Background] {:5}/{:5} Previews left to generate.",
n, number_of_tasks
)
}
match light_copy.calculate_texture_preview(&hash, format) {
Ok(preview) => {
cache.write().unwrap().insert((format, hash), preview);
}
Err(e) => {
println!("[Background] Error, Skipped Generated Preview '{}' {:?} \n\t {:?}", hash, format, e);
}
}
}
});
}
}
Ok(store)
}
@ -151,6 +206,23 @@ impl DataStore {
f.store(self.index_file_path().as_path())
}
pub fn calculate_texture_preview(
&self,
hash: &Sha256,
desired_format: TextureFormat,
) -> io::Result<Vec<u8>> {
let original = self.read_texture_file_by_hash(&hash)?;
let mut preview =
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
})?;
preview.shrink_to_fit();
Ok(preview)
}
pub fn get_texture_preview(
&mut self,
hash: &Sha256,
@ -158,18 +230,16 @@ impl DataStore {
) -> io::Result<Vec<u8>> {
let key = (desired_format.clone(), hash.clone());
if let Some(preview) = self.preview_cache.get(&key) {
if let Some(preview) = self.preview_cache.read().unwrap().get(&key) {
return Ok(preview.clone());
}
let original = self.read_texture_file_by_hash(&hash)?;
let preview = self.calculate_texture_preview(hash, desired_format)?;
let preview =
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
})?;
self.preview_cache.insert(key, preview.clone());
self.preview_cache
.write()
.unwrap()
.insert(key, preview.clone());
Ok(preview)
}