[dirty] Implement Background Preview Generation
This commit is contained in:
parent
b2ba9a9ba4
commit
47cf586116
@ -5,8 +5,9 @@ authors = ["CodeSteak <codesteak@shellf.art>"]
|
||||
edition = "2018"
|
||||
|
||||
[dependencies]
|
||||
image = "0.21.1"
|
||||
image = { version = "0.21.1", default-features = false, features = ["jpeg", "png_codec" ] }
|
||||
serde = { version = "1.0.90", features = ["derive"] }
|
||||
serde_json = "1.0.39"
|
||||
sha2 = "0.8.0"
|
||||
lovecraft = "0.2.0"
|
||||
num_cpus = "1.0"
|
@ -8,6 +8,8 @@ extern crate lovecraft;
|
||||
|
||||
extern crate sha2;
|
||||
|
||||
extern crate num_cpus;
|
||||
|
||||
pub mod model;
|
||||
pub mod persistency;
|
||||
pub mod protocol;
|
||||
|
@ -3,7 +3,7 @@
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
|
||||
#[derive(Copy, Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
|
||||
pub enum TextureFormat {
|
||||
#[serde(rename = "png")]
|
||||
PNG,
|
||||
|
@ -10,12 +10,12 @@ pub fn generate_preview(input: &[u8], format: TextureFormat) -> ImageResult<Vec<
|
||||
// Yes, this guesses the format :D
|
||||
// Also the resize function takes a maximum size and preservs the ratio.
|
||||
let img =
|
||||
::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Lanczos3);
|
||||
::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Nearest);
|
||||
|
||||
let mut out: Vec<u8> = Vec::new();
|
||||
|
||||
match format {
|
||||
TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(95))?,
|
||||
TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(85))?,
|
||||
TextureFormat::PNG => img.write_to(&mut out, ImageOutputFormat::PNG)?,
|
||||
}
|
||||
|
||||
|
@ -3,10 +3,13 @@ use crate::model::*;
|
||||
use std::collections::*;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::RwLock;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
mod image_convert;
|
||||
mod metadata_file;
|
||||
|
||||
type PreviewCache = Arc<RwLock<HashMap<(TextureFormat, Sha256), Vec<u8>>>>;
|
||||
pub struct DataStore {
|
||||
//data_path: PathBuf,
|
||||
base_dir: PathBuf,
|
||||
@ -16,7 +19,7 @@ pub struct DataStore {
|
||||
id_index: HashMap<String, Texture>,
|
||||
name_index: HashMap<String, Texture>,
|
||||
|
||||
preview_cache: HashMap<(TextureFormat, Sha256), Vec<u8>>,
|
||||
preview_cache: PreviewCache,
|
||||
}
|
||||
|
||||
impl DataStore {
|
||||
@ -38,6 +41,18 @@ impl DataStore {
|
||||
path
|
||||
}
|
||||
|
||||
fn light_clone(&self) -> DataStore {
|
||||
DataStore {
|
||||
base_dir: self.base_dir.clone(),
|
||||
|
||||
textures: Default::default(),
|
||||
id_index: Default::default(),
|
||||
name_index: Default::default(),
|
||||
|
||||
preview_cache: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
|
||||
let mut store = DataStore {
|
||||
base_dir: base_dir.into(),
|
||||
@ -51,7 +66,11 @@ impl DataStore {
|
||||
|
||||
let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?;
|
||||
|
||||
let mut preview_cache_tasks: Vec<(TextureFormat, Sha256)> = vec![];
|
||||
|
||||
for texture in metadata_file.textures.iter() {
|
||||
preview_cache_tasks.push((TextureFormat::JPEG, texture.texture_hash.clone()));
|
||||
|
||||
match store.insert(texture.clone()) {
|
||||
true => (),
|
||||
false => {
|
||||
@ -63,6 +82,42 @@ impl DataStore {
|
||||
store.garbage_collect()?;
|
||||
store.flush_metadata()?;
|
||||
|
||||
// Background Preview Generation
|
||||
{
|
||||
let number_of_tasks = preview_cache_tasks.len();
|
||||
let preview_cache_tasks = Arc::new(Mutex::new(preview_cache_tasks));
|
||||
|
||||
println!("Generating Previews");
|
||||
for _ in 0..(num_cpus::get() * 2) {
|
||||
let cache = store.preview_cache.clone();
|
||||
let light_copy = store.light_clone();
|
||||
let preview_cache_tasks = preview_cache_tasks.clone();
|
||||
|
||||
std::thread::spawn(move || {
|
||||
while let (Some((format, hash)), n) = {
|
||||
let mut tasks = preview_cache_tasks.lock().unwrap();
|
||||
(tasks.pop(), tasks.len())
|
||||
} {
|
||||
if n % 100 == 0 || n == number_of_tasks - 1 {
|
||||
println!(
|
||||
"[Background] {:5}/{:5} Previews left to generate.",
|
||||
n, number_of_tasks
|
||||
)
|
||||
}
|
||||
|
||||
match light_copy.calculate_texture_preview(&hash, format) {
|
||||
Ok(preview) => {
|
||||
cache.write().unwrap().insert((format, hash), preview);
|
||||
}
|
||||
Err(e) => {
|
||||
println!("[Background] Error, Skipped Generated Preview '{}' {:?} \n\t {:?}", hash, format, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
@ -151,6 +206,23 @@ impl DataStore {
|
||||
f.store(self.index_file_path().as_path())
|
||||
}
|
||||
|
||||
pub fn calculate_texture_preview(
|
||||
&self,
|
||||
hash: &Sha256,
|
||||
desired_format: TextureFormat,
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let original = self.read_texture_file_by_hash(&hash)?;
|
||||
|
||||
let mut preview =
|
||||
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
|
||||
})?;
|
||||
|
||||
preview.shrink_to_fit();
|
||||
|
||||
Ok(preview)
|
||||
}
|
||||
|
||||
pub fn get_texture_preview(
|
||||
&mut self,
|
||||
hash: &Sha256,
|
||||
@ -158,18 +230,16 @@ impl DataStore {
|
||||
) -> io::Result<Vec<u8>> {
|
||||
let key = (desired_format.clone(), hash.clone());
|
||||
|
||||
if let Some(preview) = self.preview_cache.get(&key) {
|
||||
if let Some(preview) = self.preview_cache.read().unwrap().get(&key) {
|
||||
return Ok(preview.clone());
|
||||
}
|
||||
|
||||
let original = self.read_texture_file_by_hash(&hash)?;
|
||||
let preview = self.calculate_texture_preview(hash, desired_format)?;
|
||||
|
||||
let preview =
|
||||
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
|
||||
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
|
||||
})?;
|
||||
|
||||
self.preview_cache.insert(key, preview.clone());
|
||||
self.preview_cache
|
||||
.write()
|
||||
.unwrap()
|
||||
.insert(key, preview.clone());
|
||||
|
||||
Ok(preview)
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user