diff options
Diffstat (limited to 'MLEB/Translate/ttmserver/TTMServerMessageUpdateJob.php')
| -rw-r--r-- | MLEB/Translate/ttmserver/TTMServerMessageUpdateJob.php | 322 |
1 files changed, 312 insertions, 10 deletions
diff --git a/MLEB/Translate/ttmserver/TTMServerMessageUpdateJob.php b/MLEB/Translate/ttmserver/TTMServerMessageUpdateJob.php index d304b857..9eb379ec 100644 --- a/MLEB/Translate/ttmserver/TTMServerMessageUpdateJob.php +++ b/MLEB/Translate/ttmserver/TTMServerMessageUpdateJob.php @@ -4,42 +4,344 @@ * * @file * @author Niklas Laxström - * @copyright Copyright © 2012-2013, Niklas Laxström - * @license GPL-2.0+ + * @license GPL-2.0-or-later */ +use MediaWiki\Logger\LoggerFactory; + /** * Job for updating translation memory. * + * job params: + * - command: the command to run, defaults to 'rebuild' + * - service: the service to write to, if set to null the job will write + * to the default (primary) service and its replicas. + * - errorCount: number of errors encountered while trying to perform the write + * on this service + * + * This job handles retries itself and return false in allowRetries to disable + * JobQueue's internal retry service. + * + * If mirroring is activated on the primary service then the first job + * will try to write to all services, it will resend a new job to + * every single service that failed and will increment errorCount. + * When too many errors occur on single service the job is dropped. + * * @ingroup JobQueue */ class TTMServerMessageUpdateJob extends Job { /** + * Number of *retries* allowed, 4 means we attempt + * to run the job 5 times (1 initial attempt + 4 retries). + */ + const MAX_ERROR_RETRY = 4; + + /** + * Constant used by backoffDelay(). + * With 7 the cumulative delay between the first and last attempt is + * between 8 and 33 minutes. + */ + const WRITE_BACKOFF_EXPONENT = 7; + + /** + * The maximum amount of time jobs delayed due to frozen services can remain + * in the job queue. + */ + const DROP_DELAYED_JOBS_AFTER = 86400; // 60 * 60 * 24 * 1; + + /** * @param MessageHandle $handle + * @param string $command * @return TTMServerMessageUpdateJob */ - public static function newJob( MessageHandle $handle ) { - $job = new self( $handle->getTitle() ); + public static function newJob( MessageHandle $handle, $command ) { + $job = new self( $handle->getTitle(), [ 'command' => $command ] ); return $job; } - function __construct( $title, $params = array(), $id = 0 ) { - parent::__construct( __CLASS__, $title, $params, $id ); + /** + * @param Title $title + * @param array $params + */ + public function __construct( $title, $params = [] ) { + parent::__construct( + __CLASS__, + $title, + $params + [ + 'command' => 'rebuild', + 'service' => null, + 'errorCount' => 0, + 'createdAt' => time(), + 'retryCount' => 0, + ] + ); } /** * Fetch all the translations and update them. + * @return bool + */ + public function run() { + global $wgTranslateTranslationServices, + $wgTranslateTranslationDefaultService; + + $service = $this->params['service']; + $writeToMirrors = false; + + if ( $service === null ) { + $service = $wgTranslateTranslationDefaultService; + $writeToMirrors = true; + } + + if ( !isset( $wgTranslateTranslationServices[$service] ) ) { + LoggerFactory::getInstance( 'TTMServerUpdates' )->warning( + 'Received update job for a an unknown service {service}.', + [ 'service' => $service ] + ); + return true; + } + + $services = [ $service ]; + if ( $writeToMirrors ) { + $config = $wgTranslateTranslationServices[$service]; + $server = TTMServer::factory( $config ); + $services = array_unique( + array_merge( $services, $server->getMirrors() ) + ); + } + + foreach ( $services as $service ) { + $this->runCommandWithRetry( $service ); + } + return true; + } + + /** + * @inheritDoc + */ + public function allowRetries() { + return false; + } + + /** + * Run the update on the specified service name. + * + * @param string $serviceName the service name + */ + private function runCommandWithRetry( $serviceName ) { + global $wgTranslateTranslationServices; + + if ( !isset( $wgTranslateTranslationServices[$serviceName] ) ) { + LoggerFactory::getInstance( 'TTMServerUpdates' )->warning( + 'Cannot write to {service}: service is unknown.', + [ 'service' => $serviceName ] + ); + return; + } + $ttmserver = TTMServer::factory( $wgTranslateTranslationServices[$serviceName] ); + + if ( $serviceName === null || !( $ttmserver instanceof WritableTTMServer ) ) { + LoggerFactory::getInstance( 'TTMServerUpdates' )->warning( + 'Received update job for a service that does not implement ' . + 'WritableTTMServer, please check config for {service}.', + [ 'service' => $serviceName ] + ); + return; + } + + try { + if ( $ttmserver->isFrozen() ) { + $this->requeueRetry( $serviceName ); + } else { + $this->runCommand( $ttmserver ); + } + } catch ( \Exception $e ) { + $this->requeueError( $serviceName, $e ); + } + } + + /** + * @param string $serviceName the service in error + * @param Exception $e the error + */ + private function requeueError( $serviceName, $e ) { + LoggerFactory::getInstance( 'TTMServerUpdates' )->warning( + 'Exception thrown while running {command} on ' . + 'service {service}: {errorMessage}', + [ + 'command' => $this->params['command'], + 'service' => $serviceName, + 'errorMessage' => $e->getMessage(), + 'exception' => $e, + ] + ); + if ( $this->params['errorCount'] >= self::MAX_ERROR_RETRY ) { + LoggerFactory::getInstance( 'TTMServerUpdates' )->warning( + 'Dropping failing job {command} for service {service} ' . + 'after repeated failure', + [ + 'command' => $this->params['command'], + 'service' => $serviceName, + ] + ); + return; + } + + $delay = self::backoffDelay( $this->params['errorCount'] ); + $job = clone $this; + $job->params['errorCount']++; + $job->params['service'] = $serviceName; + $job->setDelay( $delay ); + LoggerFactory::getInstance( 'TTMServerUpdates' )->info( + 'Update job reported failure on service {service}. ' . + 'Requeueing job with delay of {delay}.', + [ + 'service' => $serviceName, + 'delay' => $delay + ] + ); + $this->resend( $job ); + } + + /** + * Re-queue job that is frozen, or drop the job if it has + * been frozen for too long. + * + * @param string $serviceName + */ + private function requeueRetry( $serviceName ) { + $diff = time() - $this->params['createdAt']; + $dropTimeout = self::DROP_DELAYED_JOBS_AFTER; + if ( $diff > $dropTimeout ) { + LoggerFactory::getInstance( 'TTMServerUpdates' )->warning( + 'Dropping delayed job {command} for service {service} ' . + 'after waiting {diff}s', + [ + 'command' => $this->params['command'], + 'service' => $serviceName, + 'diff' => $diff, + ] + ); + } else { + $delay = self::backoffDelay( $this->params['retryCount'] ); + $job = clone $this; + $job->params['retryCount']++; + $job->params['service'] = $serviceName; + $job->setDelay( $delay ); + LoggerFactory::getInstance( 'TTMServerUpdates' )->debug( + 'Service {service} reported frozen. ' . + 'Requeueing job with delay of {delay}s', + [ + 'service' => $serviceName, + 'delay' => $delay + ] + ); + $this->resend( $job ); + } + } + + /** + * Extracted for testing purpose + * @param TTMServerMessageUpdateJob $job + */ + protected function resend( TTMServerMessageUpdateJob $job ) { + JobQueueGroup::singleton()->push( $job ); + } + + private function runCommand( WritableTTMServer $ttmserver ) { + $handle = $this->getHandle(); + $command = $this->params['command']; + + if ( $command === 'delete' ) { + $this->updateItem( $ttmserver, $handle, null, false ); + } elseif ( $command === 'rebuild' ) { + $this->updateMessage( $ttmserver, $handle ); + } elseif ( $command === 'refresh' ) { + $this->updateTranslation( $ttmserver, $handle ); + } + } + + /** + * Extracted for testing purpose + * + * @return MessageHandle + */ + protected function getHandle() { + return new MessageHandle( $this->title ); + } + + /** + * Extracted for testing purpose + * + * @param MessageHandle $handle + * @return string */ - function run() { - $handle = new MessageHandle( $this->title ); + protected function getTranslation( MessageHandle $handle ) { + return TranslateUtils::getMessageContent( + $handle->getKey(), + $handle->getCode(), + $handle->getTitle()->getNamespace() + ); + } + + private function updateMessage( WritableTTMServer $ttmserver, MessageHandle $handle ) { + // Base page update, e.g. group change. Update everything. $translations = ApiQueryMessageTranslations::getTranslations( $handle ); foreach ( $translations as $page => $data ) { $tTitle = Title::makeTitle( $this->title->getNamespace(), $page ); $tHandle = new MessageHandle( $tTitle ); - TTMServer::onChange( $tHandle, $data[0], $tHandle->isFuzzy() ); + $this->updateItem( $ttmserver, $tHandle, $data[0], $tHandle->isFuzzy() ); } + } - return true; + private function updateTranslation( WritableTTMServer $ttmserver, MessageHandle $handle ) { + // Update only this translation + $translation = $this->getTranslation( $handle ); + $this->updateItem( $ttmserver, $handle, $translation, $handle->isFuzzy() ); + } + + private function updateItem( WritableTTMServer $ttmserver, MessageHandle $handle, $text, $fuzzy ) { + if ( $fuzzy ) { + $text = null; + } + $ttmserver->update( $handle, $text ); + } + + /** + * Set a delay for this job. Note that this might not be possible, the JobQueue + * implementation handling this job doesn't support it (JobQueueDB) but is possible + * for the high performance JobQueueRedis. Note also that delays are minimums - + * at least JobQueueRedis makes no effort to remove the delay as soon as possible + * after it has expired. By default it only checks every five minutes or so. + * Note yet again that if another delay has been set that is longer then this one + * then the _longer_ delay stays. + * + * @param int $delay seconds to delay this job if possible + */ + public function setDelay( $delay ) { + $jobQueue = JobQueueGroup::singleton()->get( $this->getType() ); + if ( !$delay || !$jobQueue->delayedJobsEnabled() ) { + return; + } + $oldTime = $this->getReleaseTimestamp(); + $newTime = time() + $delay; + if ( $oldTime !== null && $oldTime >= $newTime ) { + return; + } + $this->params[ 'jobReleaseTimestamp' ] = $newTime; + } + + /** + * @param int $retryCount The number of times the job has errored out. + * @return int Number of seconds to delay. With the default minimum exponent + * of 6 the possible return values are 64, 128, 256, 512 and 1024 giving a + * maximum delay of 17 minutes. + */ + public static function backoffDelay( $retryCount ) { + return ceil( pow( + 2, + static::WRITE_BACKOFF_EXPONENT + rand( 0, min( $retryCount, 4 ) ) + ) ); } } |