Skip to content

Commit

Permalink
fix: reduce amount of type resolved by prices job (#360)
Browse files Browse the repository at this point in the history
US hosted server have a wide ping compared to EU against ESI (average 1 request every 2 seconds)
Base on this, reduce batch size to 250 (20% of previous value)
Also increase delay between job - preventing funnel usage
  • Loading branch information
warlof authored May 30, 2023
1 parent 97e9b22 commit 6adcf9c
Show file tree
Hide file tree
Showing 2 changed files with 9 additions and 7 deletions.
12 changes: 7 additions & 5 deletions src/Commands/Esi/Update/Prices.php
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ class Prices extends Command
*/
protected $description = 'Schedule updater jobs which will collect market price stats.';

const HISTORY_BATCH_SIZE = 1000;
const HISTORY_BATCH_SIZE = 250;

/**
* Execute the console command.
Expand All @@ -60,13 +60,15 @@ public function handle()
// collect all items which can be sold on the market.
$types = InvType::whereNotNull('marketGroupID')
->where('published', true)
->pluck('typeID');
->select('typeID');

//this is a guess that's only valid in the best case. In reality, we will probably be a bit slower.
$batch_processing_duration = (int) (History::ENDPOINT_RATE_LIMIT_WINDOW / History::ENDPOINT_RATE_LIMIT_CALLS * self::HISTORY_BATCH_SIZE);
$types->chunk(self::HISTORY_BATCH_SIZE)->each(function ($chunk, $index) use ($batch_processing_duration) {
$ids = $chunk->toArray();
History::dispatch($ids)->delay($index * $batch_processing_duration);

$types->chunk(self::HISTORY_BATCH_SIZE, function ($chunk, $index) use ($batch_processing_duration) {
$ids = $chunk->pluck('typeID')->toArray();

History::dispatch($ids)->delay(now()->addMinutes($index)->addSeconds($batch_processing_duration));
});
}
}
4 changes: 2 additions & 2 deletions src/Jobs/Market/History.php
Original file line number Diff line number Diff line change
Expand Up @@ -37,14 +37,14 @@ class History extends EsiBase
const THE_FORGE = 10000002;

// override the default from AbstractJob
public const JOB_EXECUTION_TIMEOUT = 60 * 60 * 24; //1 day
public const JOB_EXECUTION_TIMEOUT = 60 * 60 * 24; // 1 day

/**
* Describes how long the rate limit window lasts in seconds before resetting.
*
* @var int
*/
const ENDPOINT_RATE_LIMIT_WINDOW = 60; // to be on the safe side, we set it to 61 rather than 60
const ENDPOINT_RATE_LIMIT_WINDOW = 60;

/**
* Describes how many calls can be made in the timespan described in ENDPOINT_RATE_LIMIT_WINDOW.
Expand Down

0 comments on commit 6adcf9c

Please sign in to comment.