[dirty] Implement Background Preview Generation
This commit is contained in:
parent
b2ba9a9ba4
commit
47cf586116
|
@ -5,8 +5,9 @@ authors = ["CodeSteak <codesteak@shellf.art>"]
|
||||||
edition = "2018"
|
edition = "2018"
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
image = "0.21.1"
|
image = { version = "0.21.1", default-features = false, features = ["jpeg", "png_codec" ] }
|
||||||
serde = { version = "1.0.90", features = ["derive"] }
|
serde = { version = "1.0.90", features = ["derive"] }
|
||||||
serde_json = "1.0.39"
|
serde_json = "1.0.39"
|
||||||
sha2 = "0.8.0"
|
sha2 = "0.8.0"
|
||||||
lovecraft = "0.2.0"
|
lovecraft = "0.2.0"
|
||||||
|
num_cpus = "1.0"
|
|
@ -8,6 +8,8 @@ extern crate lovecraft;
|
||||||
|
|
||||||
extern crate sha2;
|
extern crate sha2;
|
||||||
|
|
||||||
|
extern crate num_cpus;
|
||||||
|
|
||||||
pub mod model;
|
pub mod model;
|
||||||
pub mod persistency;
|
pub mod persistency;
|
||||||
pub mod protocol;
|
pub mod protocol;
|
||||||
|
|
|
@ -3,7 +3,7 @@
|
||||||
|
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
#[derive(Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
|
#[derive(Copy, Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
|
||||||
pub enum TextureFormat {
|
pub enum TextureFormat {
|
||||||
#[serde(rename = "png")]
|
#[serde(rename = "png")]
|
||||||
PNG,
|
PNG,
|
||||||
|
|
|
@ -10,12 +10,12 @@ pub fn generate_preview(input: &[u8], format: TextureFormat) -> ImageResult<Vec<
|
||||||
// Yes, this guesses the format :D
|
// Yes, this guesses the format :D
|
||||||
// Also the resize function takes a maximum size and preservs the ratio.
|
// Also the resize function takes a maximum size and preservs the ratio.
|
||||||
let img =
|
let img =
|
||||||
::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Lanczos3);
|
::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Nearest);
|
||||||
|
|
||||||
let mut out: Vec<u8> = Vec::new();
|
let mut out: Vec<u8> = Vec::new();
|
||||||
|
|
||||||
match format {
|
match format {
|
||||||
TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(95))?,
|
TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(85))?,
|
||||||
TextureFormat::PNG => img.write_to(&mut out, ImageOutputFormat::PNG)?,
|
TextureFormat::PNG => img.write_to(&mut out, ImageOutputFormat::PNG)?,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,10 +3,13 @@ use crate::model::*;
|
||||||
use std::collections::*;
|
use std::collections::*;
|
||||||
use std::io;
|
use std::io;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::RwLock;
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
mod image_convert;
|
mod image_convert;
|
||||||
mod metadata_file;
|
mod metadata_file;
|
||||||
|
|
||||||
|
type PreviewCache = Arc<RwLock<HashMap<(TextureFormat, Sha256), Vec<u8>>>>;
|
||||||
pub struct DataStore {
|
pub struct DataStore {
|
||||||
//data_path: PathBuf,
|
//data_path: PathBuf,
|
||||||
base_dir: PathBuf,
|
base_dir: PathBuf,
|
||||||
|
@ -16,7 +19,7 @@ pub struct DataStore {
|
||||||
id_index: HashMap<String, Texture>,
|
id_index: HashMap<String, Texture>,
|
||||||
name_index: HashMap<String, Texture>,
|
name_index: HashMap<String, Texture>,
|
||||||
|
|
||||||
preview_cache: HashMap<(TextureFormat, Sha256), Vec<u8>>,
|
preview_cache: PreviewCache,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl DataStore {
|
impl DataStore {
|
||||||
|
@ -38,6 +41,18 @@ impl DataStore {
|
||||||
path
|
path
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn light_clone(&self) -> DataStore {
|
||||||
|
DataStore {
|
||||||
|
base_dir: self.base_dir.clone(),
|
||||||
|
|
||||||
|
textures: Default::default(),
|
||||||
|
id_index: Default::default(),
|
||||||
|
name_index: Default::default(),
|
||||||
|
|
||||||
|
preview_cache: Default::default(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
|
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
|
||||||
let mut store = DataStore {
|
let mut store = DataStore {
|
||||||
base_dir: base_dir.into(),
|
base_dir: base_dir.into(),
|
||||||
|
@ -51,7 +66,11 @@ impl DataStore {
|
||||||
|
|
||||||
let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?;
|
let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?;
|
||||||
|
|
||||||
|
let mut preview_cache_tasks: Vec<(TextureFormat, Sha256)> = vec![];
|
||||||
|
|
||||||
for texture in metadata_file.textures.iter() {
|
for texture in metadata_file.textures.iter() {
|
||||||
|
preview_cache_tasks.push((TextureFormat::JPEG, texture.texture_hash.clone()));
|
||||||
|
|
||||||
match store.insert(texture.clone()) {
|
match store.insert(texture.clone()) {
|
||||||
true => (),
|
true => (),
|
||||||
false => {
|
false => {
|
||||||
|
@ -63,6 +82,42 @@ impl DataStore {
|
||||||
store.garbage_collect()?;
|
store.garbage_collect()?;
|
||||||
store.flush_metadata()?;
|
store.flush_metadata()?;
|
||||||
|
|
||||||
|
// Background Preview Generation
|
||||||
|
{
|
||||||
|
let number_of_tasks = preview_cache_tasks.len();
|
||||||
|
let preview_cache_tasks = Arc::new(Mutex::new(preview_cache_tasks));
|
||||||
|
|
||||||
|
println!("Generating Previews");
|
||||||
|
for _ in 0..(num_cpus::get() * 2) {
|
||||||
|
let cache = store.preview_cache.clone();
|
||||||
|
let light_copy = store.light_clone();
|
||||||
|
let preview_cache_tasks = preview_cache_tasks.clone();
|
||||||
|
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
while let (Some((format, hash)), n) = {
|
||||||
|
let mut tasks = preview_cache_tasks.lock().unwrap();
|
||||||
|
(tasks.pop(), tasks.len())
|
||||||
|
} {
|
||||||
|
if n % 100 == 0 || n == number_of_tasks - 1 {
|
||||||
|
println!(
|
||||||
|
"[Background] {:5}/{:5} Previews left to generate.",
|
||||||
|
n, number_of_tasks
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
match light_copy.calculate_texture_preview(&hash, format) {
|
||||||
|
Ok(preview) => {
|
||||||
|
cache.write().unwrap().insert((format, hash), preview);
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
println!("[Background] Error, Skipped Generated Preview '{}' {:?} \n\t {:?}", hash, format, e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(store)
|
Ok(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -151,6 +206,23 @@ impl DataStore {
|
||||||
f.store(self.index_file_path().as_path())
|
f.store(self.index_file_path().as_path())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn calculate_texture_preview(
|
||||||
|
&self,
|
||||||
|
hash: &Sha256,
|
||||||
|
desired_format: TextureFormat,
|
||||||
|
) -> io::Result<Vec<u8>> {
|
||||||
|
let original = self.read_texture_file_by_hash(&hash)?;
|
||||||
|
|
||||||
|
let mut preview =
|
||||||
|
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
|
||||||
|
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
|
||||||
|
})?;
|
||||||
|
|
||||||
|
preview.shrink_to_fit();
|
||||||
|
|
||||||
|
Ok(preview)
|
||||||
|
}
|
||||||
|
|
||||||
pub fn get_texture_preview(
|
pub fn get_texture_preview(
|
||||||
&mut self,
|
&mut self,
|
||||||
hash: &Sha256,
|
hash: &Sha256,
|
||||||
|
@ -158,18 +230,16 @@ impl DataStore {
|
||||||
) -> io::Result<Vec<u8>> {
|
) -> io::Result<Vec<u8>> {
|
||||||
let key = (desired_format.clone(), hash.clone());
|
let key = (desired_format.clone(), hash.clone());
|
||||||
|
|
||||||
if let Some(preview) = self.preview_cache.get(&key) {
|
if let Some(preview) = self.preview_cache.read().unwrap().get(&key) {
|
||||||
return Ok(preview.clone());
|
return Ok(preview.clone());
|
||||||
}
|
}
|
||||||
|
|
||||||
let original = self.read_texture_file_by_hash(&hash)?;
|
let preview = self.calculate_texture_preview(hash, desired_format)?;
|
||||||
|
|
||||||
let preview =
|
self.preview_cache
|
||||||
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
|
.write()
|
||||||
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
|
.unwrap()
|
||||||
})?;
|
.insert(key, preview.clone());
|
||||||
|
|
||||||
self.preview_cache.insert(key, preview.clone());
|
|
||||||
|
|
||||||
Ok(preview)
|
Ok(preview)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue