localhorst/TextureSync
3
1
Fork 0
Code Issues 2 Pull Requests Releases 3 Wiki Activity

refactor persistency

Browse Source
This commit is contained in:
CodeSteak
2019-05-05 19:16:52 +02:00
parent 073eb9011c
commit 89fa985005
6 changed files with 222 additions and 218 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
server/texture-sync-server/src/model/mod.rs
View File

@ -11,7 +11,7 @@ pub use sha256::Sha256;
mod texture_format;
pub use texture_format::TextureFormat;
#[derive(Eq, PartialEq, Clone, Serialize, Deserialize, Debug)]
#[derive(Eq, PartialEq, Clone, Serialize, Deserialize, Debug, Hash)]
pub struct Texture {
pub id: String,
pub name: String,

18
server/texture-sync-server/src/model/sha256.rs
View File

@ -14,9 +14,25 @@ fn hash_to_hex_string(hash: &[u8; 32]) -> String {
}
impl Sha256 {
pub fn create_hex_string(&self) -> String {
pub fn as_hex_string(&self) -> String {
hash_to_hex_string(&self.0)
}
pub fn from_data(data: &[u8]) -> Self {
use sha2::Digest;
let mut hasher = sha2::Sha256::new();
hasher.input(data);
let hash_result = hasher.result();
let mut hash_arr = [0u8; 32];
for i in 0..32 {
hash_arr[i] = hash_result[i];
}
Sha256(hash_arr)
}
}
fn as_hex<S>(hash: &[u8; 32], serializer: S) -> Result<S::Ok, S::Error>

6
server/texture-sync-server/src/model/texture_format.rs
View File

@ -8,6 +8,12 @@ pub enum TextureFormat {
JPEG,
}
impl TextureFormat {
fn variants() -> &'static [TextureFormat] {
&[TextureFormat::PNG, TextureFormat::JPEG]
}
}
#[cfg(test)]
mod tests {
// Lol, I thought we would need custom code, like for Sha256, but it works out of the box :D

38
server/texture-sync-server/src/persistency/collection_file.rs
View File

@ -1,38 +0,0 @@
use crate::model::Texture;
use serde::{Deserialize, Serialize};
use std::fs;
use std::io;
use std::path::Path;
#[derive(Deserialize, Serialize)]
struct CollectionFile {
textures: Vec<Texture>,
}
pub fn load_collection_file(path: &Path) -> io::Result<Vec<Texture>> {
match fs::File::open(path) {
Ok(file) => {
let buf_reader = io::BufReader::new(file);
let collection: CollectionFile = serde_json::from_reader(buf_reader)?;
Ok(collection.textures)
}
Err(e) => {
if e.kind() == io::ErrorKind::NotFound {
// File has not been created yet.
Ok(Vec::new())
} else {
Err(e)
}
}
}
}
pub fn store_collection_file(path: &Path, content: &Vec<Texture>) -> io::Result<()> {
let collection_file = CollectionFile {
textures: content.clone(),
};
let file = fs::File::create(path)?;
serde_json::to_writer(file, &collection_file)?;
Ok(())
}

46
server/texture-sync-server/src/persistency/metadata_file.rs Normal file
View File

@ -0,0 +1,46 @@
use crate::model::Texture;
use serde::{Deserialize, Serialize};
use std::fs;
use std::io::{self, *};
use std::path::Path;
#[derive(Default, Deserialize, Serialize)]
pub struct MetadataFile {
pub textures: Vec<Texture>,
}
impl MetadataFile {
pub fn load(path: &Path) -> io::Result<Self> {
if !path.exists() {
return Ok(Default::default());
}
let file = fs::File::open(path)?;
let buf_reader = io::BufReader::new(file);
let collection: Self = serde_json::from_reader(buf_reader)?;
Ok(collection)
}
pub fn from_iterator<'a, I>(i: I) -> Self
where
I: Iterator<Item = &'a Texture>,
{
MetadataFile {
textures: i.map(|tex| tex.clone()).collect(),
}
}
pub fn store(self, path: &Path) -> io::Result<()> {
let mut path_tmp = path.to_path_buf();
assert!(path_tmp.set_extension("js.tmp"));
let mut file = fs::File::create(&path_tmp)?;
serde_json::to_writer(&mut file, &self)?;
file.flush()?;
drop(file);
fs::rename(path_tmp, path)
}
}

330
server/texture-sync-server/src/persistency/mod.rs
View File

@ -1,21 +1,14 @@
// TODO: remove on implementation
#![allow(unused_imports)]
#![allow(unused_variables)]
#![allow(dead_code)]
use crate::model::*;
use std::collections::HashMap;
use std::collections::*;
use std::fs;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use sha2::{self, Digest};
pub use self::search::Query;
mod collection_file;
mod image_convert;
mod metadata_file;
mod search;
pub type TextureFileResult = Result<Arc<Vec<u8>>, TextureFileError>;
@ -25,37 +18,162 @@ pub enum TextureFileError {
ImageError(::image::ImageError),
}
impl From<io::Error> for TextureFileError {
fn from(err: io::Error) -> Self {
TextureFileError::IoError(err)
}
}
impl From<::image::ImageError> for TextureFileError {
fn from(err: ::image::ImageError) -> Self {
TextureFileError::ImageError(err)
}
}
pub struct DataStore {
// private attributes
// may change
data_dir: PathBuf,
texture: Vec<Texture>,
//data_path: PathBuf,
base_dir: PathBuf,
textures: HashSet<Texture>,
id_index: HashMap<String, Texture>,
name_index: HashMap<String, Texture>,
preview_cache: HashMap<(TextureFormat, Sha256), Arc<Vec<u8>>>,
}
fn sha256(data: &[u8]) -> Sha256 {
let mut hasher = sha2::Sha256::new();
hasher.input(data);
let hash_result = hasher.result();
let hash_slice = hash_result.as_slice();
let mut hash_arr = [0u8; 32];
for i in 0..32 {
hash_arr[i] = hash_slice[i];
}
Sha256(hash_arr)
}
impl DataStore {
pub fn new(path: &Path) -> io::Result<DataStore> {
let base_path = path.to_path_buf();
let mut collection_file_path = base_path.clone();
collection_file_path.push("collection.json");
fn texture_file_path(&self, sha: &Sha256) -> PathBuf {
let mut path = self.base_dir.clone();
path.push("textures");
path.push(sha.as_hex_string());
path
}
Ok(DataStore {
data_dir: base_path,
texture: collection_file::load_collection_file(&collection_file_path)?,
preview_cache: HashMap::new(),
})
fn index_file_path(&self) -> PathBuf {
let mut path = self.base_dir.clone();
path.push("collection.json");
path
}
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
let mut store = DataStore {
base_dir: base_dir.into(),
textures: Default::default(),
id_index: Default::default(),
name_index: Default::default(),
preview_cache: Default::default(),
};
let metadata_file = metadata_file::MetadataFile::load(base_dir)?;
for texture in metadata_file.textures.iter() {
match store.insert(texture.clone())? {
true => (),
false => {
println!("inserting {:#?} failed !!!", texture); // TODO: What should be done?
}
}
}
Ok(store)
}
pub fn texture_by_id(&self, id: &str) -> Option<Texture> {
self.id_index.get(id).cloned()
}
pub fn texture_by_name(&self, id: &str) -> Option<Texture> {
self.name_index.get(id).cloned()
}
pub fn insert(&mut self, texture: Texture) -> io::Result<bool> {
if self.id_index.contains_key(&texture.id) {
return Ok(false);
}
if self.name_index.contains_key(&texture.name) {
return Ok(false);
}
if !self.is_texture_file_on_disk(&texture.texture_hash)? {
return Ok(false);
}
self.id_index.insert(texture.id.clone(), texture.clone());
self.name_index
.insert(texture.name.clone(), texture.clone());
self.textures.insert(texture.clone());
Ok(true)
}
/// returns true if successful
pub fn delete(&mut self, tex: &Texture) -> bool {
if self.textures.remove(tex) {
// remove
assert!(self.id_index.remove(&tex.id).is_some());
assert!(self.name_index.remove(&tex.name).is_some());
// don't delete cache, since it could be used
// by other texture.
true
} else {
false
}
}
/// Check if the texture, given by hash, physically exists on the file system.
pub fn is_texture_file_on_disk(&self, hash: &Sha256) -> io::Result<bool> {
let file_path = self.texture_file_path(&hash);
Ok(file_path.is_file())
}
pub fn read_texture_file_by_hash(&mut self, hash: &Sha256) -> TextureFileResult {
use std::fs::*;
use std::io::*;
if !(self.is_texture_file_on_disk(&hash)?) {
return Err(TextureFileError::NotFound);
}
let file_path = self.texture_file_path(&hash);
let mut file = File::open(file_path)?;
let mut buffer = Vec::new();
file.read_to_end(&mut buffer)?;
Ok(Arc::new(buffer))
}
pub fn store_texture_file(&mut self, data: &[u8]) -> io::Result<()> {
use std::fs;
use std::io::Write;
let hash = crate::model::Sha256::from_data(data);
let file_path = self.texture_file_path(&hash);
fs::create_dir_all(&file_path)?;
let mut file = fs::File::create(&file_path)?;
file.write_all(data)
}
fn flush_metadata(&self) -> io::Result<()> {
use std::ops::Deref;
let f = metadata_file::MetadataFile::from_iterator(self.textures.iter());
f.store(self.index_file_path().as_path())
}
pub fn get_texture_preview(&mut self, hash: &Sha256) -> TextureFileResult {
unimplemented!();
}
pub fn garbage_collect(&mut self) -> io::Result<()> {
@ -66,148 +184,4 @@ impl DataStore {
unimplemented!();
// calls self::search::search(... )
}
/// returns true if successful
pub fn delete(&mut self, tex: &Texture) -> bool {
// Find the texture
let pos = self.texture.iter().position(|e| e == tex);
match pos {
// Remove it
Some(idx) => {
let removed = self.texture.remove(idx);
let mut key = (TextureFormat::PNG, removed.texture_hash);
self.preview_cache.remove(&key); // Delete png preview
key.0 = TextureFormat::JPEG;
self.preview_cache.remove(&key); // Delete jpeg preview
true
}
// Texture not found
None => false,
}
}
/// * `data` The content of the texture file, if available.
pub fn insert(
&mut self,
tex: Texture,
data: Option<Arc<Vec<u8>>>,
) -> ProtocolResult<ReplaceTextureStatus> {
use io::Write;
// Check for collisions
if self
.texture
.iter()
.find(|e| e.id == tex.id || e.name == tex.name)
.is_some()
{
// Name or id already in use
Err(ProtocolError::Conflict(
"Name or id is already in use.".to_string(),
))
} else {
// Insert it
if self.has_hash(&tex.texture_hash)? {
self.texture.push(tex);
Ok(ReplaceTextureStatus::Ok)
} else {
match data {
None => Ok(ReplaceTextureStatus::NeedTextureData(tex.texture_hash)),
Some(blob) => {
if sha256(&blob) != tex.texture_hash {
return Err(ProtocolError::BadRequest(
"The texture does not have the given hash value.".to_string(),
));
}
let mut tmp_image_path = self.data_dir.clone();
tmp_image_path.push("textures");
tmp_image_path.push(format!(
"{}_new",
tex.texture_hash.create_hex_string().to_lowercase()
));
let mut final_image_path = self.data_dir.clone();
final_image_path.push("textures");
final_image_path.push(tex.texture_hash.create_hex_string().to_lowercase());
{
let mut writer = fs::File::create(&tmp_image_path)?;
writer.write_all(&blob)?;
}
fs::rename(tmp_image_path, final_image_path)?;
Ok(ReplaceTextureStatus::Ok)
}
}
}
}
}
pub fn by_name<'a>(&'a self, name: &str) -> Option<&'a Texture> {
self.texture.iter().find(|e| e.name == name)
}
pub fn by_id<'a, 'b>(&'a self, id: &'b str) -> Option<&'a Texture> {
self.texture.iter().find(|e| e.id == id)
}
/// Check if the texture, given by hash, physically exists on the file system.
pub fn has_hash(&self, hash: &Sha256) -> io::Result<bool> {
let mut texture_path = self.data_dir.clone();
texture_path.push("textures");
texture_path.push(hash.create_hex_string().to_lowercase());
match texture_path.metadata() {
Ok(meta) => {
if meta.is_file() {
Ok(true)
} else {
Err(io::Error::new(
io::ErrorKind::Other,
format!("{:?} exists but is not a regular file.", texture_path),
))
}
}
Err(e) => {
if e.kind() == io::ErrorKind::NotFound {
Ok(false)
} else {
Err(e)
}
}
}
}
pub fn get_texture_file(&mut self, hash: &Sha256) -> TextureFileResult {
use io::Read;
let mut file_path = self.data_dir.clone();
file_path.push("textures");
file_path.push(hash.create_hex_string().to_lowercase());
let mut file = match fs::File::open(file_path) {
Ok(f) => f,
Err(e) => {
if e.kind() == io::ErrorKind::NotFound {
return Err(TextureFileError::NotFound);
} else {
return Err(TextureFileError::IoError(e));
}
}
};
let mut buffer = Vec::new();
match file.read_to_end(&mut buffer) {
Ok(_) => (),
Err(e) => return Err(TextureFileError::IoError(e)),
};
Ok(Arc::new(buffer))
}
pub fn get_texture_preview(&mut self, hash: &Sha256) -> TextureFileResult {
unimplemented!();
}
fn store_metadata(&self) -> io::Result<()> {
let mut tmp_path = self.data_dir.clone();
tmp_path.push("collection_new.json");
collection_file::store_collection_file(&tmp_path, &self.texture)?;
let mut final_path = self.data_dir.clone();
final_path.push("collection.json");
fs::rename(tmp_path, final_path)
}
}