309 lines
9.0 KiB
Rust
309 lines
9.0 KiB
Rust
use crate::model::*;
|
|
|
|
use std::collections::*;
|
|
use std::io;
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::RwLock;
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
mod image_convert;
|
|
mod metadata_file;
|
|
|
|
type PreviewCache = Arc<RwLock<HashMap<(TextureFormat, Sha256), Vec<u8>>>>;
|
|
pub struct DataStore {
|
|
//data_path: PathBuf,
|
|
base_dir: PathBuf,
|
|
|
|
textures: HashSet<Texture>,
|
|
|
|
id_index: HashMap<String, Texture>,
|
|
name_index: HashMap<String, Texture>,
|
|
|
|
preview_cache: PreviewCache,
|
|
}
|
|
|
|
impl DataStore {
|
|
fn texture_base_path(&self) -> PathBuf {
|
|
let mut path = self.base_dir.clone();
|
|
path.push("textures");
|
|
path
|
|
}
|
|
|
|
fn texture_file_path(&self, sha: &Sha256) -> PathBuf {
|
|
let mut path = self.texture_base_path();
|
|
path.push(sha.as_hex_string());
|
|
path
|
|
}
|
|
|
|
fn index_file_path(&self) -> PathBuf {
|
|
let mut path = self.base_dir.clone();
|
|
path.push("collection.json");
|
|
path
|
|
}
|
|
|
|
fn light_clone(&self) -> DataStore {
|
|
DataStore {
|
|
base_dir: self.base_dir.clone(),
|
|
|
|
textures: Default::default(),
|
|
id_index: Default::default(),
|
|
name_index: Default::default(),
|
|
|
|
preview_cache: Default::default(),
|
|
}
|
|
}
|
|
|
|
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
|
|
let mut store = DataStore {
|
|
base_dir: base_dir.into(),
|
|
|
|
textures: Default::default(),
|
|
id_index: Default::default(),
|
|
name_index: Default::default(),
|
|
|
|
preview_cache: Default::default(),
|
|
};
|
|
|
|
let metadata_file = metadata_file::MetadataFile::load(&store.index_file_path())?;
|
|
|
|
let mut preview_cache_tasks: Vec<(TextureFormat, Sha256)> = vec![];
|
|
|
|
for texture in metadata_file.textures.iter() {
|
|
preview_cache_tasks.push((TextureFormat::JPEG, texture.texture_hash.clone()));
|
|
|
|
match store.insert(texture.clone()) {
|
|
true => (),
|
|
false => {
|
|
panic!("inserting {:#?} failed !!!", texture); // TODO: What should be done?
|
|
}
|
|
}
|
|
}
|
|
|
|
store.garbage_collect()?;
|
|
store.flush_metadata()?;
|
|
|
|
// Background Preview Generation
|
|
{
|
|
let number_of_tasks = preview_cache_tasks.len();
|
|
let preview_cache_tasks = Arc::new(Mutex::new(preview_cache_tasks));
|
|
|
|
println!("Generating Previews");
|
|
for _ in 0..(num_cpus::get() * 2) {
|
|
let cache = store.preview_cache.clone();
|
|
let light_copy = store.light_clone();
|
|
let preview_cache_tasks = preview_cache_tasks.clone();
|
|
|
|
std::thread::spawn(move || {
|
|
while let (Some((format, hash)), n) = {
|
|
let mut tasks = preview_cache_tasks.lock().unwrap();
|
|
(tasks.pop(), tasks.len())
|
|
} {
|
|
if n % 100 == 0 || n == number_of_tasks - 1 {
|
|
println!(
|
|
"[Background] {:5}/{:5} Previews left to generate.",
|
|
n, number_of_tasks
|
|
)
|
|
}
|
|
|
|
match light_copy.calculate_texture_preview(&hash, format) {
|
|
Ok(preview) => {
|
|
cache.write().unwrap().insert((format, hash), preview);
|
|
}
|
|
Err(e) => {
|
|
println!("[Background] Error, Skipped Generated Preview '{}' {:?} \n\t {:?}", hash, format, e);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
Ok(store)
|
|
}
|
|
|
|
pub fn texture_by_id(&self, id: &str) -> Option<Texture> {
|
|
self.id_index.get(id).cloned()
|
|
}
|
|
|
|
pub fn texture_by_name(&self, id: &str) -> Option<Texture> {
|
|
self.name_index.get(id).cloned()
|
|
}
|
|
|
|
pub fn insert(&mut self, texture: Texture) -> bool {
|
|
if self.id_index.contains_key(&texture.id) {
|
|
return false;
|
|
}
|
|
|
|
if self.name_index.contains_key(&texture.name) {
|
|
return false;
|
|
}
|
|
|
|
if !self.is_texture_file_on_disk(&texture.texture_hash) {
|
|
return false;
|
|
}
|
|
|
|
self.id_index.insert(texture.id.clone(), texture.clone());
|
|
self.name_index
|
|
.insert(texture.name.clone(), texture.clone());
|
|
self.textures.insert(texture.clone());
|
|
|
|
true
|
|
}
|
|
|
|
/// returns true if successful
|
|
pub fn delete(&mut self, tex: &Texture) -> bool {
|
|
if self.textures.remove(tex) {
|
|
// remove
|
|
assert!(self.id_index.remove(&tex.id).is_some());
|
|
assert!(self.name_index.remove(&tex.name).is_some());
|
|
|
|
// don't delete cache, since it could be used
|
|
// by other texture.
|
|
|
|
true
|
|
} else {
|
|
false
|
|
}
|
|
}
|
|
|
|
/// Check if the texture, given by hash, physically exists on the file system.
|
|
pub fn is_texture_file_on_disk(&self, hash: &Sha256) -> bool {
|
|
let file_path = self.texture_file_path(&hash);
|
|
|
|
file_path.is_file()
|
|
}
|
|
|
|
pub fn read_texture_file_by_hash(&self, hash: &Sha256) -> io::Result<Vec<u8>> {
|
|
use std::fs::*;
|
|
use std::io::*;
|
|
|
|
let file_path = self.texture_file_path(&hash);
|
|
|
|
let mut file = File::open(file_path)?;
|
|
let mut buffer = Vec::new();
|
|
file.read_to_end(&mut buffer)?;
|
|
|
|
Ok(buffer)
|
|
}
|
|
|
|
pub fn store_texture_file(&mut self, data: &[u8]) -> io::Result<()> {
|
|
use std::fs;
|
|
use std::io::Write;
|
|
|
|
let hash = crate::model::Sha256::from_data(data);
|
|
|
|
fs::create_dir_all(&self.texture_base_path())?;
|
|
let file_path = self.texture_file_path(&hash);
|
|
|
|
let mut file = fs::File::create(&file_path)?;
|
|
|
|
file.write_all(data)
|
|
}
|
|
|
|
pub fn flush_metadata(&self) -> io::Result<()> {
|
|
let f = metadata_file::MetadataFile::from_iterator(self.textures.iter());
|
|
|
|
f.store(self.index_file_path().as_path())
|
|
}
|
|
|
|
pub fn calculate_texture_preview(
|
|
&self,
|
|
hash: &Sha256,
|
|
desired_format: TextureFormat,
|
|
) -> io::Result<Vec<u8>> {
|
|
let original = self.read_texture_file_by_hash(&hash)?;
|
|
|
|
let mut preview =
|
|
image_convert::generate_preview(&original[..], desired_format).map_err(|_e| {
|
|
io::Error::new(io::ErrorKind::InvalidData, "Invalid Texture Image on Disk.")
|
|
})?;
|
|
|
|
preview.shrink_to_fit();
|
|
|
|
Ok(preview)
|
|
}
|
|
|
|
pub fn get_texture_preview(
|
|
&mut self,
|
|
hash: &Sha256,
|
|
desired_format: TextureFormat,
|
|
) -> io::Result<Vec<u8>> {
|
|
let key = (desired_format.clone(), hash.clone());
|
|
|
|
if let Some(preview) = self.preview_cache.read().unwrap().get(&key) {
|
|
return Ok(preview.clone());
|
|
}
|
|
|
|
let preview = self.calculate_texture_preview(hash, desired_format)?;
|
|
|
|
self.preview_cache
|
|
.write()
|
|
.unwrap()
|
|
.insert(key, preview.clone());
|
|
|
|
Ok(preview)
|
|
}
|
|
|
|
pub fn borrow_textures(&self) -> impl Iterator<Item = &Texture> {
|
|
self.textures.iter()
|
|
}
|
|
|
|
pub fn garbage_collect(&mut self) -> io::Result<()> {
|
|
fn extract_hash(filename: &std::ffi::OsStr) -> Option<Sha256> {
|
|
// directly return None for invalidly encoded file names
|
|
let str_name = filename.to_str()?;
|
|
let hash = Sha256::from_hex(str_name)?;
|
|
|
|
// check back to ignore names with lowercase letters
|
|
if hash.as_hex_string() == str_name {
|
|
Some(hash)
|
|
} else {
|
|
None
|
|
}
|
|
}
|
|
let texture_base_path = self.texture_base_path();
|
|
|
|
if !texture_base_path.exists() {
|
|
println!("No Textures Found: Skip GC.");
|
|
return Ok(());
|
|
}
|
|
|
|
let texture_dir = std::fs::read_dir(texture_base_path)?;
|
|
|
|
let mut hashs_on_disk = HashSet::new();
|
|
for result_direntry in texture_dir {
|
|
let result_direntry = result_direntry?;
|
|
|
|
// Skip if not a file.
|
|
if !result_direntry.file_type()?.is_file() {
|
|
continue;
|
|
}
|
|
|
|
let texture_path = result_direntry.path();
|
|
|
|
let filename = match texture_path.file_name() {
|
|
Some(name) => name,
|
|
None => continue,
|
|
};
|
|
match extract_hash(filename) {
|
|
Some(hash) => {
|
|
hashs_on_disk.insert(hash);
|
|
}
|
|
None => (), // ignore other files
|
|
};
|
|
}
|
|
|
|
let mut unused_files = hashs_on_disk;
|
|
for texture in &self.textures {
|
|
unused_files.remove(&texture.texture_hash);
|
|
}
|
|
|
|
// remove what is still contained in the HashSet
|
|
for entry in unused_files {
|
|
let path = self.texture_file_path(&entry);
|
|
std::fs::remove_file(path)?;
|
|
}
|
|
Ok(())
|
|
}
|
|
}
|