diff --git a/server/texture-sync-server/src/model/mod.rs b/server/texture-sync-server/src/model/mod.rs index f4b46ac..1a7ded7 100644 --- a/server/texture-sync-server/src/model/mod.rs +++ b/server/texture-sync-server/src/model/mod.rs @@ -11,7 +11,7 @@ pub use sha256::Sha256; mod texture_format; pub use texture_format::TextureFormat; -#[derive(Eq, PartialEq, Clone, Serialize, Deserialize, Debug)] +#[derive(Eq, PartialEq, Clone, Serialize, Deserialize, Debug, Hash)] pub struct Texture { pub id: String, pub name: String, diff --git a/server/texture-sync-server/src/model/sha256.rs b/server/texture-sync-server/src/model/sha256.rs index e0c5df4..5ee440d 100644 --- a/server/texture-sync-server/src/model/sha256.rs +++ b/server/texture-sync-server/src/model/sha256.rs @@ -14,9 +14,25 @@ fn hash_to_hex_string(hash: &[u8; 32]) -> String { } impl Sha256 { - pub fn create_hex_string(&self) -> String { + pub fn as_hex_string(&self) -> String { hash_to_hex_string(&self.0) } + + pub fn from_data(data: &[u8]) -> Self { + use sha2::Digest; + + let mut hasher = sha2::Sha256::new(); + hasher.input(data); + + let hash_result = hasher.result(); + + let mut hash_arr = [0u8; 32]; + for i in 0..32 { + hash_arr[i] = hash_result[i]; + } + + Sha256(hash_arr) + } } fn as_hex(hash: &[u8; 32], serializer: S) -> Result diff --git a/server/texture-sync-server/src/model/texture_format.rs b/server/texture-sync-server/src/model/texture_format.rs index 180e57c..aa24a7e 100644 --- a/server/texture-sync-server/src/model/texture_format.rs +++ b/server/texture-sync-server/src/model/texture_format.rs @@ -8,6 +8,12 @@ pub enum TextureFormat { JPEG, } +impl TextureFormat { + fn variants() -> &'static [TextureFormat] { + &[TextureFormat::PNG, TextureFormat::JPEG] + } +} + #[cfg(test)] mod tests { // Lol, I thought we would need custom code, like for Sha256, but it works out of the box :D diff --git a/server/texture-sync-server/src/persistency/collection_file.rs b/server/texture-sync-server/src/persistency/collection_file.rs deleted file mode 100644 index 2277153..0000000 --- a/server/texture-sync-server/src/persistency/collection_file.rs +++ /dev/null @@ -1,38 +0,0 @@ -use crate::model::Texture; -use serde::{Deserialize, Serialize}; -use std::fs; -use std::io; -use std::path::Path; - -#[derive(Deserialize, Serialize)] -struct CollectionFile { - textures: Vec, -} - -pub fn load_collection_file(path: &Path) -> io::Result> { - match fs::File::open(path) { - Ok(file) => { - let buf_reader = io::BufReader::new(file); - - let collection: CollectionFile = serde_json::from_reader(buf_reader)?; - Ok(collection.textures) - } - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - // File has not been created yet. - Ok(Vec::new()) - } else { - Err(e) - } - } - } -} - -pub fn store_collection_file(path: &Path, content: &Vec) -> io::Result<()> { - let collection_file = CollectionFile { - textures: content.clone(), - }; - let file = fs::File::create(path)?; - serde_json::to_writer(file, &collection_file)?; - Ok(()) -} diff --git a/server/texture-sync-server/src/persistency/metadata_file.rs b/server/texture-sync-server/src/persistency/metadata_file.rs new file mode 100644 index 0000000..81dc413 --- /dev/null +++ b/server/texture-sync-server/src/persistency/metadata_file.rs @@ -0,0 +1,46 @@ +use crate::model::Texture; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io::{self, *}; +use std::path::Path; + +#[derive(Default, Deserialize, Serialize)] +pub struct MetadataFile { + pub textures: Vec, +} + +impl MetadataFile { + pub fn load(path: &Path) -> io::Result { + if !path.exists() { + return Ok(Default::default()); + } + + let file = fs::File::open(path)?; + let buf_reader = io::BufReader::new(file); + + let collection: Self = serde_json::from_reader(buf_reader)?; + + Ok(collection) + } + + pub fn from_iterator<'a, I>(i: I) -> Self + where + I: Iterator, + { + MetadataFile { + textures: i.map(|tex| tex.clone()).collect(), + } + } + + pub fn store(self, path: &Path) -> io::Result<()> { + let mut path_tmp = path.to_path_buf(); + assert!(path_tmp.set_extension("js.tmp")); + + let mut file = fs::File::create(&path_tmp)?; + serde_json::to_writer(&mut file, &self)?; + file.flush()?; + drop(file); + + fs::rename(path_tmp, path) + } +} diff --git a/server/texture-sync-server/src/persistency/mod.rs b/server/texture-sync-server/src/persistency/mod.rs index 81108f7..90c37cc 100644 --- a/server/texture-sync-server/src/persistency/mod.rs +++ b/server/texture-sync-server/src/persistency/mod.rs @@ -1,21 +1,14 @@ -// TODO: remove on implementation -#![allow(unused_imports)] -#![allow(unused_variables)] -#![allow(dead_code)] - use crate::model::*; -use std::collections::HashMap; +use std::collections::*; use std::fs; use std::io; use std::path::{Path, PathBuf}; use std::sync::Arc; -use sha2::{self, Digest}; - pub use self::search::Query; -mod collection_file; mod image_convert; +mod metadata_file; mod search; pub type TextureFileResult = Result>, TextureFileError>; @@ -25,37 +18,162 @@ pub enum TextureFileError { ImageError(::image::ImageError), } +impl From for TextureFileError { + fn from(err: io::Error) -> Self { + TextureFileError::IoError(err) + } +} + +impl From<::image::ImageError> for TextureFileError { + fn from(err: ::image::ImageError) -> Self { + TextureFileError::ImageError(err) + } +} + pub struct DataStore { - // private attributes - // may change - data_dir: PathBuf, - texture: Vec, + //data_path: PathBuf, + base_dir: PathBuf, + + textures: HashSet, + + id_index: HashMap, + name_index: HashMap, + preview_cache: HashMap<(TextureFormat, Sha256), Arc>>, } -fn sha256(data: &[u8]) -> Sha256 { - let mut hasher = sha2::Sha256::new(); - hasher.input(data); - let hash_result = hasher.result(); - let hash_slice = hash_result.as_slice(); - let mut hash_arr = [0u8; 32]; - for i in 0..32 { - hash_arr[i] = hash_slice[i]; - } - Sha256(hash_arr) -} - impl DataStore { - pub fn new(path: &Path) -> io::Result { - let base_path = path.to_path_buf(); - let mut collection_file_path = base_path.clone(); - collection_file_path.push("collection.json"); + fn texture_file_path(&self, sha: &Sha256) -> PathBuf { + let mut path = self.base_dir.clone(); + path.push("textures"); + path.push(sha.as_hex_string()); + path + } - Ok(DataStore { - data_dir: base_path, - texture: collection_file::load_collection_file(&collection_file_path)?, - preview_cache: HashMap::new(), - }) + fn index_file_path(&self) -> PathBuf { + let mut path = self.base_dir.clone(); + path.push("collection.json"); + path + } + + pub fn new(base_dir: &Path) -> io::Result { + let mut store = DataStore { + base_dir: base_dir.into(), + + textures: Default::default(), + id_index: Default::default(), + name_index: Default::default(), + + preview_cache: Default::default(), + }; + + let metadata_file = metadata_file::MetadataFile::load(base_dir)?; + + for texture in metadata_file.textures.iter() { + match store.insert(texture.clone())? { + true => (), + false => { + println!("inserting {:#?} failed !!!", texture); // TODO: What should be done? + } + } + } + + Ok(store) + } + + pub fn texture_by_id(&self, id: &str) -> Option { + self.id_index.get(id).cloned() + } + + pub fn texture_by_name(&self, id: &str) -> Option { + self.name_index.get(id).cloned() + } + + pub fn insert(&mut self, texture: Texture) -> io::Result { + if self.id_index.contains_key(&texture.id) { + return Ok(false); + } + + if self.name_index.contains_key(&texture.name) { + return Ok(false); + } + + if !self.is_texture_file_on_disk(&texture.texture_hash)? { + return Ok(false); + } + + self.id_index.insert(texture.id.clone(), texture.clone()); + self.name_index + .insert(texture.name.clone(), texture.clone()); + self.textures.insert(texture.clone()); + + Ok(true) + } + + /// returns true if successful + pub fn delete(&mut self, tex: &Texture) -> bool { + if self.textures.remove(tex) { + // remove + assert!(self.id_index.remove(&tex.id).is_some()); + assert!(self.name_index.remove(&tex.name).is_some()); + + // don't delete cache, since it could be used + // by other texture. + + true + } else { + false + } + } + + /// Check if the texture, given by hash, physically exists on the file system. + pub fn is_texture_file_on_disk(&self, hash: &Sha256) -> io::Result { + let file_path = self.texture_file_path(&hash); + + Ok(file_path.is_file()) + } + + pub fn read_texture_file_by_hash(&mut self, hash: &Sha256) -> TextureFileResult { + use std::fs::*; + use std::io::*; + + if !(self.is_texture_file_on_disk(&hash)?) { + return Err(TextureFileError::NotFound); + } + + let file_path = self.texture_file_path(&hash); + + let mut file = File::open(file_path)?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + + Ok(Arc::new(buffer)) + } + + pub fn store_texture_file(&mut self, data: &[u8]) -> io::Result<()> { + use std::fs; + use std::io::Write; + + let hash = crate::model::Sha256::from_data(data); + let file_path = self.texture_file_path(&hash); + + fs::create_dir_all(&file_path)?; + + let mut file = fs::File::create(&file_path)?; + + file.write_all(data) + } + + fn flush_metadata(&self) -> io::Result<()> { + use std::ops::Deref; + + let f = metadata_file::MetadataFile::from_iterator(self.textures.iter()); + + f.store(self.index_file_path().as_path()) + } + + pub fn get_texture_preview(&mut self, hash: &Sha256) -> TextureFileResult { + unimplemented!(); } pub fn garbage_collect(&mut self) -> io::Result<()> { @@ -66,148 +184,4 @@ impl DataStore { unimplemented!(); // calls self::search::search(... ) } - - /// returns true if successful - pub fn delete(&mut self, tex: &Texture) -> bool { - // Find the texture - let pos = self.texture.iter().position(|e| e == tex); - match pos { - // Remove it - Some(idx) => { - let removed = self.texture.remove(idx); - let mut key = (TextureFormat::PNG, removed.texture_hash); - self.preview_cache.remove(&key); // Delete png preview - key.0 = TextureFormat::JPEG; - self.preview_cache.remove(&key); // Delete jpeg preview - true - } - // Texture not found - None => false, - } - } - - /// * `data` The content of the texture file, if available. - pub fn insert( - &mut self, - tex: Texture, - data: Option>>, - ) -> ProtocolResult { - use io::Write; - - // Check for collisions - if self - .texture - .iter() - .find(|e| e.id == tex.id || e.name == tex.name) - .is_some() - { - // Name or id already in use - Err(ProtocolError::Conflict( - "Name or id is already in use.".to_string(), - )) - } else { - // Insert it - if self.has_hash(&tex.texture_hash)? { - self.texture.push(tex); - Ok(ReplaceTextureStatus::Ok) - } else { - match data { - None => Ok(ReplaceTextureStatus::NeedTextureData(tex.texture_hash)), - Some(blob) => { - if sha256(&blob) != tex.texture_hash { - return Err(ProtocolError::BadRequest( - "The texture does not have the given hash value.".to_string(), - )); - } - let mut tmp_image_path = self.data_dir.clone(); - tmp_image_path.push("textures"); - tmp_image_path.push(format!( - "{}_new", - tex.texture_hash.create_hex_string().to_lowercase() - )); - let mut final_image_path = self.data_dir.clone(); - final_image_path.push("textures"); - final_image_path.push(tex.texture_hash.create_hex_string().to_lowercase()); - { - let mut writer = fs::File::create(&tmp_image_path)?; - writer.write_all(&blob)?; - } - fs::rename(tmp_image_path, final_image_path)?; - Ok(ReplaceTextureStatus::Ok) - } - } - } - } - } - - pub fn by_name<'a>(&'a self, name: &str) -> Option<&'a Texture> { - self.texture.iter().find(|e| e.name == name) - } - - pub fn by_id<'a, 'b>(&'a self, id: &'b str) -> Option<&'a Texture> { - self.texture.iter().find(|e| e.id == id) - } - - /// Check if the texture, given by hash, physically exists on the file system. - pub fn has_hash(&self, hash: &Sha256) -> io::Result { - let mut texture_path = self.data_dir.clone(); - texture_path.push("textures"); - texture_path.push(hash.create_hex_string().to_lowercase()); - match texture_path.metadata() { - Ok(meta) => { - if meta.is_file() { - Ok(true) - } else { - Err(io::Error::new( - io::ErrorKind::Other, - format!("{:?} exists but is not a regular file.", texture_path), - )) - } - } - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - Ok(false) - } else { - Err(e) - } - } - } - } - - pub fn get_texture_file(&mut self, hash: &Sha256) -> TextureFileResult { - use io::Read; - - let mut file_path = self.data_dir.clone(); - file_path.push("textures"); - file_path.push(hash.create_hex_string().to_lowercase()); - let mut file = match fs::File::open(file_path) { - Ok(f) => f, - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - return Err(TextureFileError::NotFound); - } else { - return Err(TextureFileError::IoError(e)); - } - } - }; - let mut buffer = Vec::new(); - match file.read_to_end(&mut buffer) { - Ok(_) => (), - Err(e) => return Err(TextureFileError::IoError(e)), - }; - Ok(Arc::new(buffer)) - } - - pub fn get_texture_preview(&mut self, hash: &Sha256) -> TextureFileResult { - unimplemented!(); - } - - fn store_metadata(&self) -> io::Result<()> { - let mut tmp_path = self.data_dir.clone(); - tmp_path.push("collection_new.json"); - collection_file::store_collection_file(&tmp_path, &self.texture)?; - let mut final_path = self.data_dir.clone(); - final_path.push("collection.json"); - fs::rename(tmp_path, final_path) - } }