refactor persistency
This commit is contained in:
parent
073eb9011c
commit
89fa985005
|
|
@ -11,7 +11,7 @@ pub use sha256::Sha256;
|
|||
mod texture_format;
|
||||
pub use texture_format::TextureFormat;
|
||||
|
||||
#[derive(Eq, PartialEq, Clone, Serialize, Deserialize, Debug)]
|
||||
#[derive(Eq, PartialEq, Clone, Serialize, Deserialize, Debug, Hash)]
|
||||
pub struct Texture {
|
||||
pub id: String,
|
||||
pub name: String,
|
||||
|
|
|
|||
|
|
@ -14,9 +14,25 @@ fn hash_to_hex_string(hash: &[u8; 32]) -> String {
|
|||
}
|
||||
|
||||
impl Sha256 {
|
||||
pub fn create_hex_string(&self) -> String {
|
||||
pub fn as_hex_string(&self) -> String {
|
||||
hash_to_hex_string(&self.0)
|
||||
}
|
||||
|
||||
pub fn from_data(data: &[u8]) -> Self {
|
||||
use sha2::Digest;
|
||||
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.input(data);
|
||||
|
||||
let hash_result = hasher.result();
|
||||
|
||||
let mut hash_arr = [0u8; 32];
|
||||
for i in 0..32 {
|
||||
hash_arr[i] = hash_result[i];
|
||||
}
|
||||
|
||||
Sha256(hash_arr)
|
||||
}
|
||||
}
|
||||
|
||||
fn as_hex<S>(hash: &[u8; 32], serializer: S) -> Result<S::Ok, S::Error>
|
||||
|
|
|
|||
|
|
@ -8,6 +8,12 @@ pub enum TextureFormat {
|
|||
JPEG,
|
||||
}
|
||||
|
||||
impl TextureFormat {
|
||||
fn variants() -> &'static [TextureFormat] {
|
||||
&[TextureFormat::PNG, TextureFormat::JPEG]
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
// Lol, I thought we would need custom code, like for Sha256, but it works out of the box :D
|
||||
|
|
|
|||
|
|
@ -1,38 +0,0 @@
|
|||
use crate::model::Texture;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
struct CollectionFile {
|
||||
textures: Vec<Texture>,
|
||||
}
|
||||
|
||||
pub fn load_collection_file(path: &Path) -> io::Result<Vec<Texture>> {
|
||||
match fs::File::open(path) {
|
||||
Ok(file) => {
|
||||
let buf_reader = io::BufReader::new(file);
|
||||
|
||||
let collection: CollectionFile = serde_json::from_reader(buf_reader)?;
|
||||
Ok(collection.textures)
|
||||
}
|
||||
Err(e) => {
|
||||
if e.kind() == io::ErrorKind::NotFound {
|
||||
// File has not been created yet.
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn store_collection_file(path: &Path, content: &Vec<Texture>) -> io::Result<()> {
|
||||
let collection_file = CollectionFile {
|
||||
textures: content.clone(),
|
||||
};
|
||||
let file = fs::File::create(path)?;
|
||||
serde_json::to_writer(file, &collection_file)?;
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
use crate::model::Texture;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fs;
|
||||
use std::io::{self, *};
|
||||
use std::path::Path;
|
||||
|
||||
#[derive(Default, Deserialize, Serialize)]
|
||||
pub struct MetadataFile {
|
||||
pub textures: Vec<Texture>,
|
||||
}
|
||||
|
||||
impl MetadataFile {
|
||||
pub fn load(path: &Path) -> io::Result<Self> {
|
||||
if !path.exists() {
|
||||
return Ok(Default::default());
|
||||
}
|
||||
|
||||
let file = fs::File::open(path)?;
|
||||
let buf_reader = io::BufReader::new(file);
|
||||
|
||||
let collection: Self = serde_json::from_reader(buf_reader)?;
|
||||
|
||||
Ok(collection)
|
||||
}
|
||||
|
||||
pub fn from_iterator<'a, I>(i: I) -> Self
|
||||
where
|
||||
I: Iterator<Item = &'a Texture>,
|
||||
{
|
||||
MetadataFile {
|
||||
textures: i.map(|tex| tex.clone()).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn store(self, path: &Path) -> io::Result<()> {
|
||||
let mut path_tmp = path.to_path_buf();
|
||||
assert!(path_tmp.set_extension("js.tmp"));
|
||||
|
||||
let mut file = fs::File::create(&path_tmp)?;
|
||||
serde_json::to_writer(&mut file, &self)?;
|
||||
file.flush()?;
|
||||
drop(file);
|
||||
|
||||
fs::rename(path_tmp, path)
|
||||
}
|
||||
}
|
||||
|
|
@ -1,21 +1,14 @@
|
|||
// TODO: remove on implementation
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::model::*;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::collections::*;
|
||||
use std::fs;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use sha2::{self, Digest};
|
||||
|
||||
pub use self::search::Query;
|
||||
mod collection_file;
|
||||
mod image_convert;
|
||||
mod metadata_file;
|
||||
mod search;
|
||||
|
||||
pub type TextureFileResult = Result<Arc<Vec<u8>>, TextureFileError>;
|
||||
|
|
@ -25,37 +18,162 @@ pub enum TextureFileError {
|
|||
ImageError(::image::ImageError),
|
||||
}
|
||||
|
||||
impl From<io::Error> for TextureFileError {
|
||||
fn from(err: io::Error) -> Self {
|
||||
TextureFileError::IoError(err)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<::image::ImageError> for TextureFileError {
|
||||
fn from(err: ::image::ImageError) -> Self {
|
||||
TextureFileError::ImageError(err)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct DataStore {
|
||||
// private attributes
|
||||
// may change
|
||||
data_dir: PathBuf,
|
||||
texture: Vec<Texture>,
|
||||
//data_path: PathBuf,
|
||||
base_dir: PathBuf,
|
||||
|
||||
textures: HashSet<Texture>,
|
||||
|
||||
id_index: HashMap<String, Texture>,
|
||||
name_index: HashMap<String, Texture>,
|
||||
|
||||
preview_cache: HashMap<(TextureFormat, Sha256), Arc<Vec<u8>>>,
|
||||
}
|
||||
|
||||
fn sha256(data: &[u8]) -> Sha256 {
|
||||
let mut hasher = sha2::Sha256::new();
|
||||
hasher.input(data);
|
||||
let hash_result = hasher.result();
|
||||
let hash_slice = hash_result.as_slice();
|
||||
let mut hash_arr = [0u8; 32];
|
||||
for i in 0..32 {
|
||||
hash_arr[i] = hash_slice[i];
|
||||
}
|
||||
Sha256(hash_arr)
|
||||
}
|
||||
|
||||
impl DataStore {
|
||||
pub fn new(path: &Path) -> io::Result<DataStore> {
|
||||
let base_path = path.to_path_buf();
|
||||
let mut collection_file_path = base_path.clone();
|
||||
collection_file_path.push("collection.json");
|
||||
fn texture_file_path(&self, sha: &Sha256) -> PathBuf {
|
||||
let mut path = self.base_dir.clone();
|
||||
path.push("textures");
|
||||
path.push(sha.as_hex_string());
|
||||
path
|
||||
}
|
||||
|
||||
Ok(DataStore {
|
||||
data_dir: base_path,
|
||||
texture: collection_file::load_collection_file(&collection_file_path)?,
|
||||
preview_cache: HashMap::new(),
|
||||
})
|
||||
fn index_file_path(&self) -> PathBuf {
|
||||
let mut path = self.base_dir.clone();
|
||||
path.push("collection.json");
|
||||
path
|
||||
}
|
||||
|
||||
pub fn new(base_dir: &Path) -> io::Result<DataStore> {
|
||||
let mut store = DataStore {
|
||||
base_dir: base_dir.into(),
|
||||
|
||||
textures: Default::default(),
|
||||
id_index: Default::default(),
|
||||
name_index: Default::default(),
|
||||
|
||||
preview_cache: Default::default(),
|
||||
};
|
||||
|
||||
let metadata_file = metadata_file::MetadataFile::load(base_dir)?;
|
||||
|
||||
for texture in metadata_file.textures.iter() {
|
||||
match store.insert(texture.clone())? {
|
||||
true => (),
|
||||
false => {
|
||||
println!("inserting {:#?} failed !!!", texture); // TODO: What should be done?
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(store)
|
||||
}
|
||||
|
||||
pub fn texture_by_id(&self, id: &str) -> Option<Texture> {
|
||||
self.id_index.get(id).cloned()
|
||||
}
|
||||
|
||||
pub fn texture_by_name(&self, id: &str) -> Option<Texture> {
|
||||
self.name_index.get(id).cloned()
|
||||
}
|
||||
|
||||
pub fn insert(&mut self, texture: Texture) -> io::Result<bool> {
|
||||
if self.id_index.contains_key(&texture.id) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if self.name_index.contains_key(&texture.name) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
if !self.is_texture_file_on_disk(&texture.texture_hash)? {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
self.id_index.insert(texture.id.clone(), texture.clone());
|
||||
self.name_index
|
||||
.insert(texture.name.clone(), texture.clone());
|
||||
self.textures.insert(texture.clone());
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// returns true if successful
|
||||
pub fn delete(&mut self, tex: &Texture) -> bool {
|
||||
if self.textures.remove(tex) {
|
||||
// remove
|
||||
assert!(self.id_index.remove(&tex.id).is_some());
|
||||
assert!(self.name_index.remove(&tex.name).is_some());
|
||||
|
||||
// don't delete cache, since it could be used
|
||||
// by other texture.
|
||||
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if the texture, given by hash, physically exists on the file system.
|
||||
pub fn is_texture_file_on_disk(&self, hash: &Sha256) -> io::Result<bool> {
|
||||
let file_path = self.texture_file_path(&hash);
|
||||
|
||||
Ok(file_path.is_file())
|
||||
}
|
||||
|
||||
pub fn read_texture_file_by_hash(&mut self, hash: &Sha256) -> TextureFileResult {
|
||||
use std::fs::*;
|
||||
use std::io::*;
|
||||
|
||||
if !(self.is_texture_file_on_disk(&hash)?) {
|
||||
return Err(TextureFileError::NotFound);
|
||||
}
|
||||
|
||||
let file_path = self.texture_file_path(&hash);
|
||||
|
||||
let mut file = File::open(file_path)?;
|
||||
let mut buffer = Vec::new();
|
||||
file.read_to_end(&mut buffer)?;
|
||||
|
||||
Ok(Arc::new(buffer))
|
||||
}
|
||||
|
||||
pub fn store_texture_file(&mut self, data: &[u8]) -> io::Result<()> {
|
||||
use std::fs;
|
||||
use std::io::Write;
|
||||
|
||||
let hash = crate::model::Sha256::from_data(data);
|
||||
let file_path = self.texture_file_path(&hash);
|
||||
|
||||
fs::create_dir_all(&file_path)?;
|
||||
|
||||
let mut file = fs::File::create(&file_path)?;
|
||||
|
||||
file.write_all(data)
|
||||
}
|
||||
|
||||
fn flush_metadata(&self) -> io::Result<()> {
|
||||
use std::ops::Deref;
|
||||
|
||||
let f = metadata_file::MetadataFile::from_iterator(self.textures.iter());
|
||||
|
||||
f.store(self.index_file_path().as_path())
|
||||
}
|
||||
|
||||
pub fn get_texture_preview(&mut self, hash: &Sha256) -> TextureFileResult {
|
||||
unimplemented!();
|
||||
}
|
||||
|
||||
pub fn garbage_collect(&mut self) -> io::Result<()> {
|
||||
|
|
@ -66,148 +184,4 @@ impl DataStore {
|
|||
unimplemented!();
|
||||
// calls self::search::search(... )
|
||||
}
|
||||
|
||||
/// returns true if successful
|
||||
pub fn delete(&mut self, tex: &Texture) -> bool {
|
||||
// Find the texture
|
||||
let pos = self.texture.iter().position(|e| e == tex);
|
||||
match pos {
|
||||
// Remove it
|
||||
Some(idx) => {
|
||||
let removed = self.texture.remove(idx);
|
||||
let mut key = (TextureFormat::PNG, removed.texture_hash);
|
||||
self.preview_cache.remove(&key); // Delete png preview
|
||||
key.0 = TextureFormat::JPEG;
|
||||
self.preview_cache.remove(&key); // Delete jpeg preview
|
||||
true
|
||||
}
|
||||
// Texture not found
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// * `data` The content of the texture file, if available.
|
||||
pub fn insert(
|
||||
&mut self,
|
||||
tex: Texture,
|
||||
data: Option<Arc<Vec<u8>>>,
|
||||
) -> ProtocolResult<ReplaceTextureStatus> {
|
||||
use io::Write;
|
||||
|
||||
// Check for collisions
|
||||
if self
|
||||
.texture
|
||||
.iter()
|
||||
.find(|e| e.id == tex.id || e.name == tex.name)
|
||||
.is_some()
|
||||
{
|
||||
// Name or id already in use
|
||||
Err(ProtocolError::Conflict(
|
||||
"Name or id is already in use.".to_string(),
|
||||
))
|
||||
} else {
|
||||
// Insert it
|
||||
if self.has_hash(&tex.texture_hash)? {
|
||||
self.texture.push(tex);
|
||||
Ok(ReplaceTextureStatus::Ok)
|
||||
} else {
|
||||
match data {
|
||||
None => Ok(ReplaceTextureStatus::NeedTextureData(tex.texture_hash)),
|
||||
Some(blob) => {
|
||||
if sha256(&blob) != tex.texture_hash {
|
||||
return Err(ProtocolError::BadRequest(
|
||||
"The texture does not have the given hash value.".to_string(),
|
||||
));
|
||||
}
|
||||
let mut tmp_image_path = self.data_dir.clone();
|
||||
tmp_image_path.push("textures");
|
||||
tmp_image_path.push(format!(
|
||||
"{}_new",
|
||||
tex.texture_hash.create_hex_string().to_lowercase()
|
||||
));
|
||||
let mut final_image_path = self.data_dir.clone();
|
||||
final_image_path.push("textures");
|
||||
final_image_path.push(tex.texture_hash.create_hex_string().to_lowercase());
|
||||
{
|
||||
let mut writer = fs::File::create(&tmp_image_path)?;
|
||||
writer.write_all(&blob)?;
|
||||
}
|
||||
fs::rename(tmp_image_path, final_image_path)?;
|
||||
Ok(ReplaceTextureStatus::Ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn by_name<'a>(&'a self, name: &str) -> Option<&'a Texture> {
|
||||
self.texture.iter().find(|e| e.name == name)
|
||||
}
|
||||
|
||||
pub fn by_id<'a, 'b>(&'a self, id: &'b str) -> Option<&'a Texture> {
|
||||
self.texture.iter().find(|e| e.id == id)
|
||||
}
|
||||
|
||||
/// Check if the texture, given by hash, physically exists on the file system.
|
||||
pub fn has_hash(&self, hash: &Sha256) -> io::Result<bool> {
|
||||
let mut texture_path = self.data_dir.clone();
|
||||
texture_path.push("textures");
|
||||
texture_path.push(hash.create_hex_string().to_lowercase());
|
||||
match texture_path.metadata() {
|
||||
Ok(meta) => {
|
||||
if meta.is_file() {
|
||||
Ok(true)
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::Other,
|
||||
format!("{:?} exists but is not a regular file.", texture_path),
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
if e.kind() == io::ErrorKind::NotFound {
|
||||
Ok(false)
|
||||
} else {
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_texture_file(&mut self, hash: &Sha256) -> TextureFileResult {
|
||||
use io::Read;
|
||||
|
||||
let mut file_path = self.data_dir.clone();
|
||||
file_path.push("textures");
|
||||
file_path.push(hash.create_hex_string().to_lowercase());
|
||||
let mut file = match fs::File::open(file_path) {
|
||||
Ok(f) => f,
|
||||
Err(e) => {
|
||||
if e.kind() == io::ErrorKind::NotFound {
|
||||
return Err(TextureFileError::NotFound);
|
||||
} else {
|
||||
return Err(TextureFileError::IoError(e));
|
||||
}
|
||||
}
|
||||
};
|
||||
let mut buffer = Vec::new();
|
||||
match file.read_to_end(&mut buffer) {
|
||||
Ok(_) => (),
|
||||
Err(e) => return Err(TextureFileError::IoError(e)),
|
||||
};
|
||||
Ok(Arc::new(buffer))
|
||||
}
|
||||
|
||||
pub fn get_texture_preview(&mut self, hash: &Sha256) -> TextureFileResult {
|
||||
unimplemented!();
|
||||
}
|
||||
|
||||
fn store_metadata(&self) -> io::Result<()> {
|
||||
let mut tmp_path = self.data_dir.clone();
|
||||
tmp_path.push("collection_new.json");
|
||||
collection_file::store_collection_file(&tmp_path, &self.texture)?;
|
||||
let mut final_path = self.data_dir.clone();
|
||||
final_path.push("collection.json");
|
||||
fs::rename(tmp_path, final_path)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue