localhorst/reHDD
1
0
Fork 1
Code Issues 4 Pull Requests Releases 12 Wiki Activity

add cold start

Browse Source
This commit is contained in:
Hendrik Schutter
2026-05-03 08:41:34 +02:00
parent 05a3750b03
commit 4ae0a89ccd
2 changed files with 67 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
include/shred.h
+2
View File
@@ -26,6 +26,7 @@
#define CHUNK_SIZE_STEP_UP 1024 * 1024 * 4 // Increase step: 4MB (symmetric with step down) #define CHUNK_SIZE_STEP_UP 1024 * 1024 * 4 // Increase step: 4MB (symmetric with step down)
#define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB (symmetric exploration) #define CHUNK_SIZE_STEP_DOWN 1024 * 1024 * 4 // Decrease step: 4MB (symmetric exploration)
#define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks #define CHUNK_MEASURE_INTERVAL 64 // Measure performance every 64 chunks
#define WARMUP_MEASUREMENTS 16 // Skip first 16 measurements (cache writes)
// Multi-armed bandit exploration parameters // Multi-armed bandit exploration parameters
#define EXPLORATION_EPSILON 0.10 // 10% exploration rate (epsilon-greedy) #define EXPLORATION_EPSILON 0.10 // 10% exploration rate (epsilon-greedy)
@@ -67,6 +68,7 @@ private:
size_t bestChunkSize; size_t bestChunkSize;
unsigned int chunkCounter; unsigned int chunkCounter;
unsigned int totalChunkCounter; // Total chunks written (for periodic re-exploration) unsigned int totalChunkCounter; // Total chunks written (for periodic re-exploration)
unsigned int warmupCounter; // Count warm-up measurements
std::chrono::high_resolution_clock::time_point measurementStartTime; std::chrono::high_resolution_clock::time_point measurementStartTime;
double bestThroughputMBps; double bestThroughputMBps;
double lastThroughputMBps; double lastThroughputMBps;
src/shred.cpp
+65 -28
View File
@@ -44,6 +44,7 @@ Shred::Shred()
bestChunkSize = CHUNK_SIZE_START; bestChunkSize = CHUNK_SIZE_START;
chunkCounter = 0; chunkCounter = 0;
totalChunkCounter = 0; // Track total chunks for periodic re-exploration totalChunkCounter = 0; // Track total chunks for periodic re-exploration
warmupCounter = 0; // Track warm-up measurements
bestThroughputMBps = 0.0; bestThroughputMBps = 0.0;
lastThroughputMBps = 0.0; lastThroughputMBps = 0.0;
bytesWrittenInMeasurement = 0; bytesWrittenInMeasurement = 0;
@@ -55,9 +56,12 @@ Shred::Shred()
Logger::logThis()->info("Adaptive chunk size optimization ENABLED (Multi-Armed Bandit) - Starting with " + Logger::logThis()->info("Adaptive chunk size optimization ENABLED (Multi-Armed Bandit) - Starting with " +
to_string(currentChunkSize / (1024 * 1024)) + " MB chunks"); to_string(currentChunkSize / (1024 * 1024)) + " MB chunks");
Logger::logThis()->info("Exploration strategy: " + to_string((int)(EXPLORATION_EPSILON * 100)) + Logger::logThis()->info("Configuration: min=" + to_string(CHUNK_SIZE_MIN / (1024 * 1024)) +
"% epsilon-greedy + periodic re-exploration every " + "MB, max=" + to_string(CHUNK_SIZE_MAX / (1024 * 1024)) +
to_string(REEXPLORATION_INTERVAL) + " chunks"); "MB, step=" + to_string(CHUNK_SIZE_STEP_UP / (1024 * 1024)) + "MB");
Logger::logThis()->info("Exploration: " + to_string((int)(EXPLORATION_EPSILON * 100)) +
"% epsilon-greedy + periodic every " + to_string(REEXPLORATION_INTERVAL) + " chunks");
Logger::logThis()->info("Warm-up: First " + to_string(WARMUP_MEASUREMENTS) + " measurements ignored (cold start protection)");
} }
Shred::~Shred() Shred::~Shred()
@@ -102,31 +106,48 @@ void Shred::evaluateThroughput(Drive *drive)
double throughputMBps = (bytesWrittenInMeasurement / (1024.0 * 1024.0)) / elapsedSeconds; double throughputMBps = (bytesWrittenInMeasurement / (1024.0 * 1024.0)) / elapsedSeconds;
lastThroughputMBps = throughputMBps; lastThroughputMBps = throughputMBps;
Logger::logThis()->info("Throughput measurement - ChunkSize: " + // Warm-up period - ignore first measurements
to_string(currentChunkSize / (1024 * 1024)) + " MB, " + warmupCounter++;
"Throughput: " + to_string((int)throughputMBps) + " MB/s, " + bool isWarmup = (warmupCounter <= WARMUP_MEASUREMENTS);
"Best: " + to_string((int)bestThroughputMBps) + " MB/s" +
" - Drive: " + drive->getSerial());
// Check if this is better than our best if (isWarmup)
if (throughputMBps > bestThroughputMBps)
{ {
bestThroughputMBps = throughputMBps; Logger::logThis()->info("WARM-UP #" + to_string(warmupCounter) + "/" + to_string(WARMUP_MEASUREMENTS) +
bestChunkSize = currentChunkSize; " - ChunkSize: " + to_string(currentChunkSize / (1024 * 1024)) + " MB, " +
throughputIncreasing = true; "Throughput: " + to_string((int)throughputMBps) + " MB/s (not used for optimization)" +
" - Drive: " + drive->getSerial());
Logger::logThis()->info("NEW BEST throughput: " + to_string((int)bestThroughputMBps) +
" MB/s with " + to_string(currentChunkSize / (1024 * 1024)) +
" MB chunks - Drive: " + drive->getSerial());
} }
else else
{ {
throughputIncreasing = false; Logger::logThis()->info("Throughput measurement - ChunkSize: " +
to_string(currentChunkSize / (1024 * 1024)) + " MB, " +
"Throughput: " + to_string((int)throughputMBps) + " MB/s, " +
"Best: " + to_string((int)bestThroughputMBps) + " MB/s" +
" - Drive: " + drive->getSerial());
// Check if this is better than our best (only after warm-up)
if (throughputMBps > bestThroughputMBps)
{
bestThroughputMBps = throughputMBps;
bestChunkSize = currentChunkSize;
throughputIncreasing = true;
Logger::logThis()->info("NEW BEST throughput: " + to_string((int)bestThroughputMBps) +
" MB/s with " + to_string(currentChunkSize / (1024 * 1024)) +
" MB chunks - Drive: " + drive->getSerial());
}
else
{
throughputIncreasing = false;
}
} }
} }
// Adjust chunk size for next measurement interval // Adjust chunk size for next measurement interval (skip during warm-up)
adjustChunkSize(drive); if (warmupCounter > WARMUP_MEASUREMENTS)
{
adjustChunkSize(drive);
}
// Start new measurement // Start new measurement
startMeasurement(); startMeasurement();
@@ -158,12 +179,12 @@ void Shred::performExploration(Drive *drive)
{ {
size_t savedChunkSize = currentChunkSize; size_t savedChunkSize = currentChunkSize;
// Generate random chunk size between MIN and MAX (aligned to 4MB boundaries) // Generate random chunk size between MIN and MAX (aligned to STEP boundaries)
size_t numSteps = (CHUNK_SIZE_MAX - CHUNK_SIZE_MIN) / CHUNK_SIZE_STEP_UP; size_t numSteps = (CHUNK_SIZE_MAX - CHUNK_SIZE_MIN) / CHUNK_SIZE_STEP_UP;
size_t randomStep = rand() % (numSteps + 1); size_t randomStep = rand() % (numSteps + 1);
explorationChunkSize = CHUNK_SIZE_MIN + (randomStep * CHUNK_SIZE_STEP_UP); explorationChunkSize = CHUNK_SIZE_MIN + (randomStep * CHUNK_SIZE_STEP_UP);
// Clamp to valid range // Clamp to valid range (safety check)
if (explorationChunkSize < CHUNK_SIZE_MIN) if (explorationChunkSize < CHUNK_SIZE_MIN)
explorationChunkSize = CHUNK_SIZE_MIN; explorationChunkSize = CHUNK_SIZE_MIN;
if (explorationChunkSize > CHUNK_SIZE_MAX) if (explorationChunkSize > CHUNK_SIZE_MAX)
@@ -173,10 +194,12 @@ void Shred::performExploration(Drive *drive)
explorationMode = true; explorationMode = true;
currentChunkSize = explorationChunkSize; currentChunkSize = explorationChunkSize;
// Enhanced logging with debug info
Logger::logThis()->info("EXPLORATION MODE: Testing " + Logger::logThis()->info("EXPLORATION MODE: Testing " +
to_string(explorationChunkSize / (1024 * 1024)) + " MB chunks " + to_string(explorationChunkSize / (1024 * 1024)) + " MB chunks " +
"(was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, best: " + "(randomStep=" + to_string(randomStep) + "/" + to_string(numSteps) + ", " +
to_string(bestChunkSize / (1024 * 1024)) + " MB)" + "was " + to_string(savedChunkSize / (1024 * 1024)) + " MB, " +
"best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
" - Drive: " + drive->getSerial()); " - Drive: " + drive->getSerial());
} }
@@ -200,10 +223,22 @@ void Shred::adjustChunkSize(Drive *drive)
if (explorationMode) if (explorationMode)
{ {
explorationMode = false; explorationMode = false;
currentChunkSize = bestChunkSize; // Return to best known chunk size
Logger::logThis()->info("EXPLORATION ENDED - Returning to best known: " + // CRITICAL: Return to best known chunk size, not current
to_string(bestChunkSize / (1024 * 1024)) + " MB" + if (currentChunkSize != bestChunkSize)
" - Drive: " + drive->getSerial()); {
currentChunkSize = bestChunkSize;
Logger::logThis()->info("EXPLORATION ENDED - Returning to best known: " +
to_string(bestChunkSize / (1024 * 1024)) + " MB" +
" (exploration tested " + to_string(oldChunkSize / (1024 * 1024)) + " MB)" +
" - Drive: " + drive->getSerial());
}
else
{
Logger::logThis()->info("EXPLORATION ENDED - Staying at current best: " +
to_string(bestChunkSize / (1024 * 1024)) + " MB" +
" - Drive: " + drive->getSerial());
}
return; return;
} }
@@ -236,6 +271,7 @@ void Shred::adjustChunkSize(Drive *drive)
currentChunkSize = CHUNK_SIZE_MIN; currentChunkSize = CHUNK_SIZE_MIN;
Logger::logThis()->info("Reached minimum chunk size: " + Logger::logThis()->info("Reached minimum chunk size: " +
to_string(currentChunkSize / (1024 * 1024)) + " MB" + to_string(currentChunkSize / (1024 * 1024)) + " MB" +
" (best remains: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
" - Drive: " + drive->getSerial()); " - Drive: " + drive->getSerial());
} }
} }
@@ -245,6 +281,7 @@ void Shred::adjustChunkSize(Drive *drive)
Logger::logThis()->info("Adjusted chunk size: " + Logger::logThis()->info("Adjusted chunk size: " +
to_string(oldChunkSize / (1024 * 1024)) + " MB -> " + to_string(oldChunkSize / (1024 * 1024)) + " MB -> " +
to_string(currentChunkSize / (1024 * 1024)) + " MB" + to_string(currentChunkSize / (1024 * 1024)) + " MB" +
" (best: " + to_string(bestChunkSize / (1024 * 1024)) + " MB)" +
" - Drive: " + drive->getSerial()); " - Drive: " + drive->getSerial());
} }
} }