Merge pull request 'speedup shred' (#46) from speedup into develop
Reviewed-on: #46
This commit is contained in:
commit
3ee59d8c58
|
|
@ -45,4 +45,6 @@ reHDD
|
|||
*.ods
|
||||
*.txt
|
||||
|
||||
.vscode/
|
||||
|
||||
ignoreDrives.conf
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
[submodule "tfnoisegen"]
|
||||
path = tfnoisegen
|
||||
url = https://git.mosad.xyz/localhorst/tfnoisegen.git
|
||||
|
|
@ -11,6 +11,8 @@
|
|||
## Debian Build Notes
|
||||
|
||||
* `apt-get install ncurses-dev git make g++`
|
||||
* `git submodule init`
|
||||
* `git submodule update`
|
||||
* `make release`
|
||||
|
||||
## Create Standalone with Debian 11
|
||||
|
|
@ -29,6 +31,11 @@ Instructions how to create a standalone machine that boots directly to reHDD. Th
|
|||
|
||||
clone this repo into /root/
|
||||
|
||||
```
|
||||
git submodule init
|
||||
git submodule update
|
||||
```
|
||||
|
||||
`cd /root/reHDD/`
|
||||
|
||||
`make release`
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ public:
|
|||
time_t u32ShredTimeDelta;
|
||||
std::chrono::time_point<std::chrono::system_clock> chronoShredTimestamp;
|
||||
unsigned long ulWrittenBytes;
|
||||
unsigned long ulSpeedMetricBytesWritten;
|
||||
} sShredSpeed;
|
||||
|
||||
bool bWasShredded = false;
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@
|
|||
#define WORSE_POWERUP 10000 //mark drive if at this limit or beyond
|
||||
#define SHRED_ITERATIONS 3U
|
||||
#define FROZEN_TIMEOUT 10 //After this timeout (minutes) the drive will be marked as frozen
|
||||
#define METRIC_THRESHOLD 3L*1000L*1000L*1000L //calc shred speed with this minimum of time delta
|
||||
|
||||
// Logger Settings
|
||||
#define LOG_PATH "./reHDD.log"
|
||||
|
|
@ -84,6 +85,7 @@ private:
|
|||
static void searchDrives(list <Drive>* plistDrives);
|
||||
static void printDrives(list <Drive>* plistDrives);
|
||||
static void startShredAllDrives(list <Drive>* plistDrives);
|
||||
static void updateShredMetrics(list <Drive>* plistDrives);
|
||||
static void filterIgnoredDrives(list <Drive>* plistDrives);
|
||||
static void filterNewDrives(list <Drive>* plistOldDrives, list <Drive>* plistNewDrives);
|
||||
static void addSMARTData(list <Drive>* plistDrives);
|
||||
|
|
|
|||
|
|
@ -17,8 +17,9 @@
|
|||
#include <unistd.h>
|
||||
#include <string.h>
|
||||
|
||||
#define CHUNK_SIZE 1024*1024*2 //amount of bytes that are overwritten at once --> 2MB
|
||||
#define CHUNK_DIMENSION 100U //amount of chunks are read at once from random source
|
||||
|
||||
#define CHUNK_SIZE 1024*1024*32 //amount of bytes that are overwritten at once --> 32MB
|
||||
#define TFNG_DATA_SIZE CHUNK_SIZE //amount of bytes used by tfng
|
||||
|
||||
//#define DEMO_DRIVE_SIZE 1024*1024*256L // 256MB
|
||||
//#define DEMO_DRIVE_SIZE 1024*1024*1024L // 1GB
|
||||
|
|
@ -39,7 +40,8 @@ public:
|
|||
private:
|
||||
fileDescriptor randomSrcFileDiscr;
|
||||
fileDescriptor driveFileDiscr;
|
||||
unsigned char caChunk[CHUNK_DIMENSION][CHUNK_SIZE];
|
||||
unsigned char caTfngData[TFNG_DATA_SIZE];
|
||||
unsigned char caReadBuffer[CHUNK_SIZE];
|
||||
unsigned long ulDriveByteSize;
|
||||
unsigned long ulDriveByteOverallCount = 0; //all bytes shredded in all iterations + checking -> used for progress calculation
|
||||
double d32Percent = 0.0;
|
||||
|
|
|
|||
16
makefile
16
makefile
|
|
@ -8,7 +8,7 @@ SRC_EXT = cpp
|
|||
# Path to the source directory, relative to the makefile
|
||||
SRC_PATH = src
|
||||
# Space-separated pkg-config libraries used by this project
|
||||
LIBS =
|
||||
LIBS = lib
|
||||
# General compiler flags
|
||||
COMPILE_FLAGS = -std=c++17 -Wall -Wextra -g
|
||||
# Additional release-specific flags
|
||||
|
|
@ -18,9 +18,12 @@ DCOMPILE_FLAGS = -D DEBUG
|
|||
# Add additional include paths
|
||||
INCLUDES = include
|
||||
# General linker settings
|
||||
LINK_FLAGS = -lpthread -lncurses
|
||||
LINK_FLAGS = -Llib -lpthread -lncurses -ltfng
|
||||
|
||||
# Doc
|
||||
DOCDIR = doc
|
||||
TFRANDDIR = tfnoisegen
|
||||
TFRANDLIB = libtfng.a
|
||||
#### END PROJECT SETTINGS ####
|
||||
|
||||
# Optionally you may move the section above to a separate config.mk file, and
|
||||
|
|
@ -51,7 +54,7 @@ ifeq ($(V),true)
|
|||
endif
|
||||
|
||||
# Combine compiler and linker flags
|
||||
release: export CXXFLAGS := $(CXXFLAGS) $(COMPILE_FLAGS) $(RCOMPILE_FLAGS)
|
||||
release: export CXXFLAGS := $(CXXFLAGS) $(COMPILE_FLAGS) $(RCOMPILE_FLAGS)
|
||||
release: export LDFLAGS := $(LDFLAGS) $(LINK_FLAGS) $(RLINK_FLAGS)
|
||||
debug: export CXXFLAGS := $(CXXFLAGS) $(COMPILE_FLAGS) $(DCOMPILE_FLAGS)
|
||||
debug: export LDFLAGS := $(LDFLAGS) $(LINK_FLAGS) $(DLINK_FLAGS)
|
||||
|
|
@ -158,6 +161,7 @@ dirs:
|
|||
@echo "Creating directories"
|
||||
@mkdir -p $(dir $(OBJECTS))
|
||||
@mkdir -p $(BIN_PATH)
|
||||
@mkdir -p $(LIBS)
|
||||
|
||||
# Removes all build files
|
||||
.PHONY: clean
|
||||
|
|
@ -167,16 +171,22 @@ clean:
|
|||
@echo "Deleting directories"
|
||||
@$(RM) -r build
|
||||
@$(RM) -r bin
|
||||
@$(RM) -r $(LIBS)
|
||||
@$(RM) -f reHDD.log
|
||||
$(MAKE) clean -C tfnoisegen
|
||||
|
||||
# Main rule, checks the executable and symlinks to the output
|
||||
all: $(BIN_PATH)/$(BIN_NAME)
|
||||
$(MAKE) libtfng.a -C tfnoisegen
|
||||
@cp $(TFRANDDIR)/$(TFRANDLIB) $(LIBS)
|
||||
@echo "Making symlink: $(BIN_NAME) -> $<"
|
||||
@$(RM) $(BIN_NAME)
|
||||
@ln -s $(BIN_PATH)/$(BIN_NAME) $(BIN_NAME)
|
||||
|
||||
# Link the executable
|
||||
$(BIN_PATH)/$(BIN_NAME): $(OBJECTS)
|
||||
$(MAKE) libtfng.a -C tfnoisegen
|
||||
@cp $(TFRANDDIR)/$(TFRANDLIB) $(LIBS)
|
||||
@echo "Linking: $@"
|
||||
@$(START_TIME)
|
||||
$(CMD_PREFIX)$(CXX) $(OBJECTS) $(LDFLAGS) -o $@
|
||||
|
|
|
|||
|
|
@ -71,9 +71,14 @@ void reHDD::app_logic(void)
|
|||
{
|
||||
char dummy;
|
||||
read (fdShredInformPipe[0],&dummy,1);
|
||||
updateShredMetrics(&listDrives);
|
||||
#ifdef LOG_LEVEL_HIGH
|
||||
Logger::logThis()->info("got progress signal from a shred task");
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
ui->updateTUI(&listDrives, u8SelectedEntry);
|
||||
} //endless loop
|
||||
|
|
@ -402,7 +407,7 @@ void reHDD::startShredAllDrives(list <Drive>* plistDrives)
|
|||
ostringstream address;
|
||||
address << (void const *)&(*pTmpDrive);
|
||||
Logger::logThis()->info("Started shred (all) for: " + pTmpDrive->getModelName() + "-" + pTmpDrive->getSerial() + " @" + address.str());
|
||||
#endif
|
||||
#endif
|
||||
pTmpDrive->state = Drive::TaskState::SHRED_ACTIVE;
|
||||
thread(ThreadShred, pTmpDrive).detach();
|
||||
}
|
||||
|
|
@ -445,6 +450,33 @@ void reHDD::printDrives(list <Drive>* plistDrives)
|
|||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief update shred metrics for all drives
|
||||
* \param pointer of list <Drive>* plistDrives
|
||||
* \return void
|
||||
*/
|
||||
void reHDD::updateShredMetrics(list <Drive>* plistDrives)
|
||||
{
|
||||
list <Drive>::iterator it;
|
||||
for (it = plistDrives->begin(); it != plistDrives->end(); ++it)
|
||||
{
|
||||
if(it->state == Drive::SHRED_ACTIVE)
|
||||
{
|
||||
Drive* pTmpDrive = iterator_to_pointer<Drive, std::list<Drive>::iterator > (it);
|
||||
//set metrics for calculating shred speed
|
||||
std::chrono::time_point<std::chrono::system_clock> chronoCurrentTimestamp = std::chrono::system_clock::now();
|
||||
time_t u32ShredTimeDelta = (chronoCurrentTimestamp - pTmpDrive->sShredSpeed.chronoShredTimestamp).count();
|
||||
if(u32ShredTimeDelta > METRIC_THRESHOLD)
|
||||
{
|
||||
pTmpDrive->sShredSpeed.u32ShredTimeDelta = u32ShredTimeDelta;
|
||||
pTmpDrive->sShredSpeed.chronoShredTimestamp = std::chrono::system_clock::now();
|
||||
pTmpDrive->sShredSpeed.ulWrittenBytes = pTmpDrive->sShredSpeed.ulSpeedMetricBytesWritten;
|
||||
pTmpDrive->sShredSpeed.ulSpeedMetricBytesWritten = 0U;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief add S.M.A.R.T data from SMART
|
||||
* \param pointer of list <Drive>* plistDrives
|
||||
|
|
|
|||
|
|
@ -2,11 +2,19 @@
|
|||
* @file shred.cpp
|
||||
* @brief shred drive
|
||||
* @author hendrik schutter
|
||||
* @date 03.05.2020
|
||||
* @date 22.08.2022
|
||||
*/
|
||||
|
||||
#include "../include/reHDD.h"
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
#include "../tfnoisegen/tfprng.h"
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
const static char *randomsrc = (char*) "/dev/urandom";
|
||||
|
||||
Shred::Shred()
|
||||
|
|
@ -43,6 +51,7 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
|
||||
#ifndef DRYRUN
|
||||
const char *cpDrivePath = drive->getPath().c_str();
|
||||
unsigned char ucKey[TFNG_KEY_SIZE];
|
||||
|
||||
//open random source
|
||||
randomSrcFileDiscr = open(randomsrc, O_RDONLY | O_LARGEFILE);
|
||||
|
|
@ -66,9 +75,22 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
return -1;
|
||||
}
|
||||
|
||||
//read key for random generator
|
||||
ssize_t readRet = read(randomSrcFileDiscr, ucKey, sizeof(ucKey)) ;
|
||||
if (readRet <= 0)
|
||||
{
|
||||
std::string errorMsg(strerror(readRet));
|
||||
Logger::logThis()->error("Shred-Task: Read random key failed! " + errorMsg + " - Drive: " + drive->getSerial());
|
||||
perror(randomsrc);
|
||||
cleanup();
|
||||
return -1;
|
||||
}
|
||||
|
||||
tfng_prng_seedkey(ucKey);
|
||||
|
||||
this->ulDriveByteSize = getDriveSizeInBytes(driveFileDiscr);
|
||||
drive->sShredSpeed.chronoShredTimestamp = std::chrono::system_clock::now();; //set inital timestamp for speed metric
|
||||
unsigned long ulSpeedMetricBytesWritten = 0U; //uses to calculate speed metric
|
||||
drive->sShredSpeed.ulSpeedMetricBytesWritten = 0U; //uses to calculate speed metric
|
||||
|
||||
#ifdef LOG_LEVEL_HIGH
|
||||
Logger::logThis()->info("Shred-Task: Bytes-Size of Drive: " + to_string(this->ulDriveByteSize) + " - Drive: " + drive->getSerial());
|
||||
|
|
@ -77,45 +99,21 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
for (unsigned int uiShredIterationCounter = 0U; uiShredIterationCounter < SHRED_ITERATIONS; uiShredIterationCounter++)
|
||||
{
|
||||
unsigned long ulDriveByteCounter = 0U; //used for one shred-iteration to keep track of the current drive position
|
||||
uint32_t u32ChunkDimensionIndex = 0U;
|
||||
|
||||
|
||||
if(uiShredIterationCounter == (SHRED_ITERATIONS-1))
|
||||
{
|
||||
//last shred iteration --> overwrite with zeros instead with random data
|
||||
memset(caChunk, 0U, CHUNK_DIMENSION*CHUNK_SIZE);
|
||||
//last shred iteration --> overwrite (just the write chunk) bytes with zeros instead with random data
|
||||
memset(caTfngData, 0U, CHUNK_SIZE);
|
||||
}
|
||||
|
||||
while (ulDriveByteCounter < ulDriveByteSize)
|
||||
{
|
||||
int iBytesToShred = 0; //Bytes that will be overwritten in this chunk-iteration
|
||||
|
||||
if((u32ChunkDimensionIndex == 0U) && (uiShredIterationCounter != (SHRED_ITERATIONS-1)))
|
||||
if(uiShredIterationCounter != (SHRED_ITERATIONS-1))
|
||||
{
|
||||
//read new chunks from random source if needed and this is NOT the last shred iteration
|
||||
unsigned long ulBytesInChunkBuffer = 0U;
|
||||
|
||||
while (ulBytesInChunkBuffer < CHUNK_DIMENSION*CHUNK_SIZE)
|
||||
{
|
||||
//read new random bytes
|
||||
int iReadBytes = read(randomSrcFileDiscr, caChunk, ((CHUNK_DIMENSION*CHUNK_SIZE)-ulBytesInChunkBuffer));
|
||||
if (iReadBytes > 0)
|
||||
{
|
||||
ulBytesInChunkBuffer += iReadBytes;
|
||||
}
|
||||
else
|
||||
{
|
||||
std::string errorMsg(strerror(iReadBytes));
|
||||
Logger::logThis()->error("Shred-Task: Read from random source failed! " + errorMsg + " - Drive: " + drive->getSerial());
|
||||
perror("unable to read random data");
|
||||
cleanup();
|
||||
return -1;;
|
||||
}
|
||||
} //end chunk read
|
||||
#ifdef LOG_LEVEL_HIGH
|
||||
Logger::logThis()->info("Shred-Task: Read new random data - Drive: " + drive->getSerial());
|
||||
#endif
|
||||
|
||||
//NOT last shred iteration --> generate new random data
|
||||
tfng_prng_genrandom(caTfngData, TFNG_DATA_SIZE);
|
||||
}
|
||||
|
||||
if((ulDriveByteSize-ulDriveByteCounter) < CHUNK_SIZE)
|
||||
|
|
@ -127,7 +125,7 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
iBytesToShred = CHUNK_SIZE;
|
||||
}
|
||||
|
||||
int iByteShredded = write(driveFileDiscr, caChunk[u32ChunkDimensionIndex], iBytesToShred);
|
||||
int iByteShredded = write(driveFileDiscr, caTfngData, iBytesToShred);
|
||||
|
||||
if(iByteShredded <= 0)
|
||||
{
|
||||
|
|
@ -138,11 +136,10 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
return -1;
|
||||
}
|
||||
|
||||
u32ChunkDimensionIndex = (u32ChunkDimensionIndex+1)%CHUNK_DIMENSION;
|
||||
ulDriveByteCounter += iByteShredded;
|
||||
ulDriveByteOverallCount += iByteShredded;
|
||||
d32Percent = this->calcProgress();
|
||||
ulSpeedMetricBytesWritten += iByteShredded;
|
||||
drive->sShredSpeed.ulSpeedMetricBytesWritten += iByteShredded;
|
||||
|
||||
#ifdef LOG_LEVEL_HIGH
|
||||
Logger::logThis()->info("Shred-Task: ByteCount: " + to_string(ulDriveByteCounter) + " - iteration: " + to_string((uiShredIterationCounter+1)) + " - progress: " + to_string(d32Percent) + " - Drive: " + drive->getSerial());
|
||||
|
|
@ -152,14 +149,6 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
//set shred percantage
|
||||
drive->setTaskPercentage(d32TmpPercent);
|
||||
d32TmpPercent = d32Percent;
|
||||
|
||||
//set metrics for calculating shred speed
|
||||
std::chrono::time_point<std::chrono::system_clock> chronoCurrentTimestamp = std::chrono::system_clock::now();
|
||||
drive->sShredSpeed.u32ShredTimeDelta = (chronoCurrentTimestamp - drive->sShredSpeed.chronoShredTimestamp).count();
|
||||
drive->sShredSpeed.chronoShredTimestamp = std::chrono::system_clock::now();
|
||||
drive->sShredSpeed.ulWrittenBytes = ulSpeedMetricBytesWritten;
|
||||
ulSpeedMetricBytesWritten = 0U;
|
||||
|
||||
//signal process in shreding
|
||||
write(*ipSignalFd, "A",1);
|
||||
}
|
||||
|
|
@ -174,13 +163,19 @@ int Shred::shredDrive(Drive* drive, int* ipSignalFd)
|
|||
cleanup();
|
||||
return -1;
|
||||
}
|
||||
}//end one chunk write
|
||||
//end one chunk write
|
||||
}
|
||||
if(0 != iRewindDrive(driveFileDiscr))
|
||||
{
|
||||
Logger::logThis()->error("Shred-Task: Unable to rewind drive! - Drive: " + drive->getSerial());
|
||||
cleanup();
|
||||
return -1;
|
||||
}
|
||||
} //end one shred iteration
|
||||
//end one shred iteration
|
||||
}
|
||||
//end of all shred iteratio
|
||||
|
||||
tfng_prng_seedkey(NULL); //reset random generator
|
||||
|
||||
#ifdef ZERO_CHECK_ALERT
|
||||
drive->u32DriveChecksumAferShredding = uiCalcChecksum(driveFileDiscr, drive, ipSignalFd);
|
||||
|
|
@ -267,10 +262,10 @@ unsigned int Shred::uiCalcChecksum(fileDescriptor file,Drive* drive, int* ipSign
|
|||
{
|
||||
iBytesToCheck = CHUNK_SIZE;
|
||||
}
|
||||
int iReadBytes = read(file, caChunk, iBytesToCheck);
|
||||
int iReadBytes = read(file, caReadBuffer, iBytesToCheck);
|
||||
for (int iReadBytesCounter = 0U; iReadBytesCounter < iReadBytes; iReadBytesCounter++)
|
||||
{
|
||||
uiChecksum += caChunk[0][iReadBytesCounter];
|
||||
uiChecksum += caReadBuffer[iReadBytesCounter];
|
||||
}
|
||||
ulDriveByteCounter += iReadBytes;
|
||||
ulDriveByteOverallCount += iReadBytes;
|
||||
|
|
@ -291,5 +286,5 @@ unsigned int Shred::uiCalcChecksum(fileDescriptor file,Drive* drive, int* ipSign
|
|||
void Shred::cleanup()
|
||||
{
|
||||
close(driveFileDiscr);
|
||||
close( randomSrcFileDiscr);
|
||||
close(randomSrcFileDiscr);
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 488716ef22ac5a1aae235a59bea2997ac7e8e45a
|
||||
Loading…
Reference in New Issue