Implement adaptive chunk sizes / adaptive speed benchmark results

src/inc/api/APISendProgress.php

@@ -495,6 +495,11 @@ public function execute(array $QUERY = array()): void {
         // the chunk has finished (exhausted)
         $chunk = Factory::getChunkFactory()->mset($chunk, [Chunk::SPEED => 0, Chunk::PROGRESS => 10000, Chunk::CHECKPOINT => $chunk->getSkip() + $chunk->getLength()]);
         DServerLog::log(DServerLog::TRACE, "Chunk is exhausted (cracker status)", [$this->agent, $chunk]);
+
+        // If we don't make use of static chunks we attempt to tune the duration a chunk takes to calculate
+        if($task->getStaticChunks() === 0 && SConfig::getInstance()->getVal(DConfig::CHUNK_DURATION_AUTO_TUNE)) {
+          TaskUtils::tuneChunkDuration($chunk, $task, $this->agent);
+        }
         break;
       case DHashcatStatus::CRACKED:
         // the chunk has finished (cracked whole hashList)

src/inc/defines/DConfig.php

@@ -7,25 +7,26 @@
 
 class DConfig {
   // Section: Cracking/Tasks
-  const BENCHMARK_TIME         = "benchtime";
-  const CHUNK_DURATION         = "chunktime";
-  const CHUNK_TIMEOUT          = "chunktimeout";
-  const AGENT_TIMEOUT          = "agenttimeout";
-  const FIELD_SEPARATOR        = "fieldseparator";
-  const HASHLIST_ALIAS         = "hashlistAlias";
-  const STATUS_TIMER           = "statustimer";
-  const BLACKLIST_CHARS        = "blacklistChars";
-  const DISP_TOLERANCE         = "disptolerance";
-  const DEFAULT_BENCH          = "defaultBenchmark";
-  const AGENT_DATA_LIFETIME    = "agentDataLifetime";
-  const DISABLE_TRIMMING       = "disableTrimming";
-  const PRIORITY_0_START       = "priority0Start";
-  const HASHCAT_BRAIN_ENABLE   = "hashcatBrainEnable";
-  const HASHCAT_BRAIN_HOST     = "hashcatBrainHost";
-  const HASHCAT_BRAIN_PORT     = "hashcatBrainPort";
-  const HASHCAT_BRAIN_PASS     = "hashcatBrainPass";
-  const HASHLIST_IMPORT_CHECK  = "hashlistImportCheck";
-  const HC_ERROR_IGNORE        = "hcErrorIgnore";
+  const BENCHMARK_TIME            = "benchtime";
+  const CHUNK_DURATION            = "chunktime";
+  const CHUNK_DURATION_AUTO_TUNE  = "chunktimeAutoTune";
+  const CHUNK_TIMEOUT             = "chunktimeout";
+  const AGENT_TIMEOUT             = "agenttimeout";
+  const FIELD_SEPARATOR           = "fieldseparator";
+  const HASHLIST_ALIAS            = "hashlistAlias";
+  const STATUS_TIMER              = "statustimer";
+  const BLACKLIST_CHARS           = "blacklistChars";
+  const DISP_TOLERANCE            = "disptolerance";
+  const DEFAULT_BENCH             = "defaultBenchmark";
+  const AGENT_DATA_LIFETIME       = "agentDataLifetime";
+  const DISABLE_TRIMMING          = "disableTrimming";
+  const PRIORITY_0_START          = "priority0Start";
+  const HASHCAT_BRAIN_ENABLE      = "hashcatBrainEnable";
+  const HASHCAT_BRAIN_HOST        = "hashcatBrainHost";
+  const HASHCAT_BRAIN_PORT        = "hashcatBrainPort";
+  const HASHCAT_BRAIN_PASS        = "hashcatBrainPass";
+  const HASHLIST_IMPORT_CHECK     = "hashlistImportCheck";
+  const HC_ERROR_IGNORE           = "hcErrorIgnore";
 
   // Section: Yubikey
   const YUBIKEY_ID  = "yubikey_id";
@@ -122,6 +123,7 @@ public static function getConfigType(string $config): string {
     return match ($config) {
       DConfig::BENCHMARK_TIME => DConfigType::NUMBER_INPUT,
       DConfig::CHUNK_DURATION => DConfigType::NUMBER_INPUT,
+      DConfig::CHUNK_DURATION_AUTO_TUNE => DConfigType::TICKBOX,
       DConfig::CHUNK_TIMEOUT => DConfigType::NUMBER_INPUT,
       DConfig::AGENT_TIMEOUT => DConfigType::NUMBER_INPUT,
       DConfig::HASHES_PAGE_SIZE => DConfigType::NUMBER_INPUT,
@@ -190,6 +192,7 @@ public static function getConfigDescription(string $config): string {
     return match ($config) {
       DConfig::BENCHMARK_TIME => "Time in seconds an agent should benchmark a task.",
       DConfig::CHUNK_DURATION => "Time in seconds a client should be working on a single chunk.",
+      DConfig::CHUNK_DURATION_AUTO_TUNE => "Automatically adjust chunk sizes to get the actual chunk working time for a client as close as possible to the configured chunk duration.",
       DConfig::CHUNK_TIMEOUT => "Time in seconds the server will consider an issued chunk as inactive or timed out and will reallocate to another client.",
       DConfig::AGENT_TIMEOUT => "Time in seconds the server will consider a client inactive or timed out.",
       DConfig::HASHES_PAGE_SIZE => "Number of hashes shown on each page of the hashes view.",

src/inc/utils/TaskUtils.php

@@ -44,6 +44,7 @@
 use Hashtopolis\inc\defines\DPrince;
 use Hashtopolis\inc\defines\DTaskStaticChunking;
 use Hashtopolis\inc\defines\DTaskTypes;
+use Hashtopolis\inc\defines\DServerLog;
 use Hashtopolis\inc\handlers\NotificationHandler;
 use Hashtopolis\inc\HTException;
 use Hashtopolis\inc\SConfig;
@@ -1519,4 +1520,42 @@ public static function getCurrentSpeedOfTask(int $taskId, ?int $agentId = null):
     $agg = new Aggregation(Chunk::SPEED, Aggregation::SUM);
     return (int)(Factory::getChunkFactory()->multicolAggregationFilter([Factory::FILTER => array_filter([$qF1, $qF2, $qF3, $qF4])], [$agg])[$agg->getName()] ?? 0);
   }
+
+  /**
+   *  Adjust the benchmark of an agent for a task to ensure the actual time it takes to calculate a chunk
+   *  is within 20% of the configured chunk duration time.
+   *
+   * @param Chunk $chunk
+   * @param Task $task
+   * @throws Exception
+   */
+  public static function tuneChunkDuration(Chunk $chunk, Task $task, Agent $agent): void {
+    $timeTaken = $chunk->getSolveTime() - $chunk->getDispatchTime();
+    if($timeTaken < 0) return; // prevent math & logic errors
+
+    $differenceToChunk = $task->getChunkTime() / $timeTaken;
+    if ($differenceToChunk < 1.2) return; // if the difference is less than 20% don't make any adjustments, margin of error. We also disregard any 
+                                          // adjustments that would result in a smaller chunk size (anything < 1.0), since we do not want to perform
+                                          // automatic reductions in chunk size.
+
+    // Limit the multiplier to 1.5, this prevents overshooting the time an agent should work on a chunk.
+    // This value could be increased (to allow for faster ramping up), but since the benchmark results and/or 
+    // chunk calculation times for small chunks can be pretty inaccurate this is a pretty safe and reasonable
+    // multiplier. Keep in mind the chunk duration will not be tuned down, only up. So once overshot, you're
+    // stuck with that time for the remainder of the task.
+    $differenceToChunk = ($differenceToChunk > 1.5) ? 1.5 : $differenceToChunk;
+
+    $qF1 = new QueryFilter(Assignment::AGENT_ID, $chunk->getAgentId(), "=");
+    $qF2 = new QueryFilter(Assignment::TASK_ID, $chunk->getTaskId(), "=");
+    $assignment = Factory::getAssignmentFactory()->filter([Factory::FILTER => [$qF1, $qF2]])[0];
+
+    $benchmark = $assignment->getBenchmark();
+    $benchmarkParts = explode(":", $benchmark);
+    if($benchmarkParts[0] == 0 || count($benchmarkParts) != 2) return;
+    $newBenchmark = $differenceToChunk * $benchmarkParts[0];
+    $assignment->setBenchmark(round($newBenchmark).":".round($benchmarkParts[1]));
+    DServerLog::log(DServerLog::INFO, "{$timeTaken}---{$task->getChunkTime()}", [$agent, $assignment]);
+    DServerLog::log(DServerLog::INFO, "Multiplied the benchmark of agent by ".round($differenceToChunk,2), [$agent, $assignment]);
+    Factory::getAssignmentFactory()->update($assignment);
+  }
 }

src/migrations/mysql/20260810144313_chunk-auto-tune-config.sql

@@ -0,0 +1,2 @@
+INSERT INTO `Config` (`configId`, `configSectionId`, `item`, `value`) VALUES
+  (80, 1, 'chunktimeAutoTune', '0');

src/migrations/postgres.1/20260810144313_chunk-auto-tune-config.sql

@@ -0,0 +1,2 @@
+INSERT INTO `Config` (`configId`, `configSectionId`, `item`, `value`) VALUES
+  (80, 1, 'chunktimeAutoTune', '0');