[dirty] Implement Background Preview Generation

server/texture-sync-server/Cargo.toml

@@ -5,8 +5,9 @@ authors = ["CodeSteak <codesteak@shellf.art>"]
 edition = "2018"
 
 [dependencies]
-image = "0.21.1"
+image = { version = "0.21.1", default-features = false, features = ["jpeg", "png_codec" ] }
 serde = { version = "1.0.90", features = ["derive"] }
 serde_json = "1.0.39"
 sha2 = "0.8.0"
 lovecraft = "0.2.0"
+num_cpus = "1.0"

server/texture-sync-server/src/main.rs

@@ -8,6 +8,8 @@ extern crate lovecraft;
 
 extern crate sha2;
 
+extern crate num_cpus;
+
 pub mod model;
 pub mod persistency;
 pub mod protocol;

server/texture-sync-server/src/model/texture_format.rs

@@ -3,7 +3,7 @@
 
 use serde::{Deserialize, Serialize};
 
-#[derive(Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, Eq, Hash, PartialEq, PartialOrd, Ord)]
 pub enum TextureFormat {
     #[serde(rename = "png")]
     PNG,

server/texture-sync-server/src/persistency/image_convert/mod.rs

@@ -10,12 +10,12 @@ pub fn generate_preview(input: &[u8], format: TextureFormat) -> ImageResult<Vec<
     // Yes, this guesses the format :D
     // Also the resize function takes a maximum size and preservs the ratio.
     let img =
-        ::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Lanczos3);
+        ::image::load_from_memory(input)?.resize(RESIZE, RESIZE, ::image::FilterType::Nearest);
 
     let mut out: Vec<u8> = Vec::new();
 
     match format {
-        TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(95))?,
+        TextureFormat::JPEG => img.write_to(&mut out, ImageOutputFormat::JPEG(85))?,
         TextureFormat::PNG => img.write_to(&mut out, ImageOutputFormat::PNG)?,
     }
 

server/texture-sync-server/src/persistency/mod.rs

@@ -3,10 +3,13 @@ use crate::model::*;
 use std::collections::*;
 use std::io;
 use std::path::{Path, PathBuf};
+use std::sync::RwLock;
+use std::sync::{Arc, Mutex};
 
 mod image_convert;
 mod metadata_file;
 
+type PreviewCache = Arc<RwLock<HashMap<(TextureFormat, Sha256), Vec<u8>>>>;
 pub struct DataStore {
     //data_path: PathBuf,
     base_dir: PathBuf,
@@ -16,7 +19,7 @@ pub struct DataStore {
     id_index: HashMap<String, Texture>,
     name_index: HashMap<String, Texture>,
 
-    preview_cache: HashMap<(TextureFormat, Sha256), Vec<u8>>,
+    preview_cache: PreviewCache,
 }
 
 impl DataStore {
@@ -38,6 +41,18 @@ impl DataStore {
         path
     }
 
+    fn light_clone(&self) -> DataStore {
+        DataStore {
+            base_dir: self.base_dir.clone(),
+
+            textures: Default::default(),
+            id_index: Default::default(),
+            name_index: Default::default(),
+
+            preview_cache: Default::default(),
+        }
+    }
+
     pub fn new(base_dir: &Path) -> io::Result<DataStore> {
         let mut store = DataStore {
             base_dir: base_dir.into(),
@@ -51,7 +66,11 @@ impl DataStore {
 
         let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?;
 
+        let mut preview_cache_tasks: Vec<(TextureFormat, Sha256)> = vec![];
+
         for texture in metadata_file.textures.iter() {
+            preview_cache_tasks.push((TextureFormat::JPEG, texture.texture_hash.clone()));
+
             match store.insert(texture.clone()) {
                 true => (),
                 false => {
@@ -63,6 +82,42 @@ impl DataStore {
         store.garbage_collect()?;
         store.flush_metadata()?;
 
+        // Background Preview Generation
+        {
+            let number_of_tasks = preview_cache_tasks.len();
+            let preview_cache_tasks = Arc::new(Mutex::new(preview_cache_tasks));
+
+            println!("Generating Previews");
+            for _ in 0..(num_cpus::get() * 2) {
+                let cache = store.preview_cache.clone();
+                let light_copy = store.light_clone();
+                let preview_cache_tasks = preview_cache_tasks.clone();
+
+                std::thread::spawn(move || {
+                    while let (Some((format, hash)), n) = {
+                        let mut tasks = preview_cache_tasks.lock().unwrap();
+                        (tasks.pop(), tasks.len())
+                    } {
+                        if n % 100 == 0 || n == number_of_tasks - 1 {
+                            println!(
+                                "[Background] {:5}/{:5} Previews left to generate.",
+                                n, number_of_tasks
+                            )
+                        }
+
+                        match light_copy.calculate_texture_preview(&hash, format) {
+                            Ok(preview) => {
+                                cache.write().unwrap().insert((format, hash), preview);
+                            }
+                            Err(e) => {
+                                println!("[Background] Error, Skipped Generated Preview '{}' {:?} \n\t {:?}", hash, format, e);
+                            }
+                        }
+                    }
+                });
+            }
+        }
+
         Ok(store)
     }
 
@@ -151,6 +206,23 @@ impl DataStore {
         f.store(self.index_file_path().as_path())
     }
 
+    pub fn calculate_texture_preview(
+        &self,
+        hash: &Sha256,
+        desired_format: TextureFormat,
+    ) -> io::Result<Vec<u8>> {
+        let original = self.read_texture_file_by_hash(&hash)?;
+
+        let mut preview =
+            image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
+                io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
+            })?;
+
+        preview.shrink_to_fit();
+
+        Ok(preview)
+    }
+
     pub fn get_texture_preview(
         &mut self,
         hash: &Sha256,
@@ -158,18 +230,16 @@ impl DataStore {
     ) -> io::Result<Vec<u8>> {
         let key = (desired_format.clone(), hash.clone());
 
-        if let Some(preview) = self.preview_cache.get(&key) {
+        if let Some(preview) = self.preview_cache.read().unwrap().get(&key) {
             return Ok(preview.clone());
         }
 
-        let original = self.read_texture_file_by_hash(&hash)?;
+        let preview = self.calculate_texture_preview(hash, desired_format)?;
 
-        let preview =
+        self.preview_cache
-            image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
+            .write()
-                io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
+            .unwrap()
-            })?;
+            .insert(key, preview.clone());
-
-        self.preview_cache.insert(key, preview.clone());
 
         Ok(preview)
     }