From d25c49228c1aecbe55fb08bb0508a3f6f19e9c54 Mon Sep 17 00:00:00 2001 From: Josh Richards Date: Sun, 23 Jun 2024 11:03:10 -0400 Subject: [PATCH] feat(updater): download progress logging and resume Signed-off-by: Josh Richards --- index.php | 82 ++++++++++++++++++++++++++++++++++++++++-------- lib/Updater.php | 82 ++++++++++++++++++++++++++++++++++++++++-------- updater.phar | Bin 1173225 -> 1175497 bytes 3 files changed, 138 insertions(+), 26 deletions(-) diff --git a/index.php b/index.php index 1fe13825..48665577 100644 --- a/index.php +++ b/index.php @@ -567,30 +567,52 @@ private function getUpdateServerResponse(): array { /** * Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename * + * Logs download progress + * Resumes incomplete downloads if possible + * Supports outbound proxy usage + * Logs download statistics upon completion + * + * TODO: Provide download progress in real-time (in both CLI and Web modes) + * * @throws \Exception */ public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); $response = $this->getUpdateServerResponse(); - - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); - } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); - } - if (!isset($response['url']) || !is_string($response['url'])) { throw new \Exception('Response from update server is missing url'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + $saveLocation = $storageLocation . basename($response['url']); + $ch = curl_init($response['url']); + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + //$this->silentLog('[debug] sleeping for 10s to enable manual RANGE while testing'); + //sleep(10); + } else { + $this->silentLog('[info] storage location already exists'); + // see if there's an existing incomplete download to resume + if (is_file($saveLocation)) { + $size = filesize($saveLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + } + + $fp = fopen($saveLocation, 'a'); curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => array($this, 'downloadProgressCallback'), CURLOPT_FILE => $fp, CURLOPT_USERAGENT => 'Nextcloud Updater', ]); @@ -607,7 +629,7 @@ public function downloadUpdate(): void { throw new \Exception('Curl error: ' . curl_error($ch)); } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { + if ($httpCode !== 200 && $httpCode !== 206) { $statusCodes = [ 400 => 'Bad request', 401 => 'Unauthorized', @@ -634,13 +656,47 @@ public function downloadUpdate(): void { $message .= ' - URL: ' . htmlentities($response['url']); throw new \Exception($message); + } else { + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog("[info] download stats: size=" . $this->formatBytes($info['size_download']) . " bytes; total_time=" . round($info['total_time'], 2) . " secs; avg speed=" . $this->formatBytes($info['speed_download']) . "/sec"); } + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); } + private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + static $previousProgress = 0; + + if ($download_size !== 0) { + $progress = round($downloaded * 100 / $download_size); + if ($progress > $previousProgress) { + $previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = array('B', 'KB', 'MB', 'GB', 'TB'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[$pow]; + } + /** * @throws \Exception */ diff --git a/lib/Updater.php b/lib/Updater.php index a7e61722..57144633 100644 --- a/lib/Updater.php +++ b/lib/Updater.php @@ -529,30 +529,52 @@ private function getUpdateServerResponse(): array { /** * Downloads the nextcloud folder to $DATADIR/updater-$instanceid/downloads/$filename * + * Logs download progress + * Resumes incomplete downloads if possible + * Supports outbound proxy usage + * Logs download statistics upon completion + * + * TODO: Provide download progress in real-time (in both CLI and Web modes) + * * @throws \Exception */ public function downloadUpdate(): void { $this->silentLog('[info] downloadUpdate()'); $response = $this->getUpdateServerResponse(); - - $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; - if (file_exists($storageLocation)) { - $this->silentLog('[info] storage location exists'); - $this->recursiveDelete($storageLocation); - } - $state = mkdir($storageLocation, 0750, true); - if ($state === false) { - throw new \Exception('Could not mkdir storage location'); - } - if (!isset($response['url']) || !is_string($response['url'])) { throw new \Exception('Response from update server is missing url'); } - $fp = fopen($storageLocation . basename($response['url']), 'w+'); + $storageLocation = $this->getUpdateDirectoryLocation() . '/updater-'.$this->getConfigOptionMandatoryString('instanceid') . '/downloads/'; + $saveLocation = $storageLocation . basename($response['url']); + $ch = curl_init($response['url']); + + if (!file_exists($storageLocation)) { + $state = mkdir($storageLocation, 0750, true); + if ($state === false) { + throw new \Exception('Could not mkdir storage location'); + } + $this->silentLog('[info] storage location created'); + //$this->silentLog('[debug] sleeping for 10s to enable manual RANGE while testing'); + //sleep(10); + } else { + $this->silentLog('[info] storage location already exists'); + // see if there's an existing incomplete download to resume + if (is_file($saveLocation)) { + $size = filesize($saveLocation); + $range = $size . '-'; + curl_setopt($ch, CURLOPT_RANGE, $range); + $this->silentLog('[info] previous download found; resuming from ' . $this->formatBytes($size)); + } + } + + $fp = fopen($saveLocation, 'a'); curl_setopt_array($ch, [ + CURLOPT_RETURNTRANSFER => true, + CURLOPT_NOPROGRESS => false, + CURLOPT_PROGRESSFUNCTION => array($this, 'downloadProgressCallback'), CURLOPT_FILE => $fp, CURLOPT_USERAGENT => 'Nextcloud Updater', ]); @@ -569,7 +591,7 @@ public function downloadUpdate(): void { throw new \Exception('Curl error: ' . curl_error($ch)); } $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE); - if ($httpCode !== 200) { + if ($httpCode !== 200 && $httpCode !== 206) { $statusCodes = [ 400 => 'Bad request', 401 => 'Unauthorized', @@ -596,13 +618,47 @@ public function downloadUpdate(): void { $message .= ' - URL: ' . htmlentities($response['url']); throw new \Exception($message); + } else { + // download succeeded + $info = curl_getinfo($ch); + $this->silentLog("[info] download stats: size=" . $this->formatBytes($info['size_download']) . " bytes; total_time=" . round($info['total_time'], 2) . " secs; avg speed=" . $this->formatBytes($info['speed_download']) . "/sec"); } + curl_close($ch); fclose($fp); $this->silentLog('[info] end of downloadUpdate()'); } + private function downloadProgressCallback(\CurlHandle $resource, int $download_size, int $downloaded, int $upload_size, int $uploaded): void { + static $previousProgress = 0; + + if ($download_size !== 0) { + $progress = round($downloaded * 100 / $download_size); + if ($progress > $previousProgress) { + $previousProgress = $progress; + // log every 2% increment for the first 10% then only log every 10% increment after that + if ($progress % 10 === 0 || ($progress < 10 && $progress % 2 === 0)) { + $this->silentLog("[info] download progress: $progress% (" . $this->formatBytes($downloaded) . " of " . $this->formatBytes($download_size) . ")"); + } + } + } + } + + private function formatBytes(int $bytes, int $precision = 2): string { + $units = array('B', 'KB', 'MB', 'GB', 'TB'); + + $bytes = max($bytes, 0); + $pow = floor(($bytes ? log($bytes) : 0) / log(1024)); + $pow = min($pow, count($units) - 1); + + // Uncomment one of the following alternatives + $bytes /= pow(1024, $pow); + // $bytes /= (1 << (10 * $pow)); + + return round($bytes, $precision) . $units[$pow]; + } + /** * @throws \Exception */ diff --git a/updater.phar b/updater.phar index 4a38828cc0af74490154147eb84ec8d19751da8d..5661b68769877c72c59cc7729272ebb79cd35b29 100755 GIT binary patch delta 2561 zcmai0U2GIp6t-)X?gG*afv6aAY_`lU?6R{97;U?)v_e5j3#E-nZJODgyW0t~b2oGE z^arUc^1y?%NNjo&V?yGq#@~nfZ1jN#6G?pVMSV2-BoGrHJ@?M;_6LYf&TQx2^LM}V zopa~U9|!OJKKQ}yZ3S8?Sx0TNWLws)PY-W=|7&!^`!^?Be$BUEdLsN&zf1QHJoxZv z-?{egkHSUu`|;fy>1!)}*H`*lP$DQ#qO_tsh4M7Y7L+!WC`t@vE6O&M?I_Qn>_BN> zUFjpOtvhqsblEoPD|Xfd+wsM`A@BB*oHWmn`0AIxk^2!@Ya{z)W_sH}3GZy*BX@ka zwQY5Lki7X?ZHSdQ*lgBwnPo%OXJwyq9*e>O7^OU@P!5h)WRW9BOQg#DygsE^MdBA*eDB>MQjRw1fhmX;1Ly6$zY|iz$$aJOE`S0goz~L>yb$&;ccZ5jdSNRMVv|&>Zt!W>1hC> zao_UFN-b1B1V-8jcZz}UPH`$&RV3oY=@bl%j}DET9h=e^q#(4^ht>F=uKIMwVZl0b zmK3eJ!>l!|KC6HMhafJd9iHw*m@Ag(pGPDyIZQDbmSsV{yF~W1ud8*P6k z`zC3V+tuUZb+WT2{D7v^fg9-!kDMJH85})+{ygM+@$^`R4f~DV6XU}JW2Z)jm6qlE z)_g)|g^?P~g*?{1HDI}J!75%dk{cYII5l(}R-eB~NMtwc-w*L=A*ut&UwAQ}hn)G+ z>cLy&LF84=?OtT?@2gM?#HnmmqamOn$tbzu;Op#Y5rchUSD0Fw#IwsWG2s%2SK#5m$tGgUO9VurOwK4C|!2 zZev7UYqQ>SXc#gE4eCfX(xW!c=d5W$W>d-`?T!_GyPxvUIy zhWhi6>r}(Urxl85skw&Z21<_41*X%f-aKHQI}dKHX`?pQjI5HN%FGgau4oLJ2kk^J zkVn3oaODbu)7%asE1i%FHJc+UTAu?PCzK~vCP?eUG;f%9t$FH%#N!CR26sqS zRzj*jdfqftH7m4svLh+)UnM&u3o&`@6YRP5^GTCUH%*;WScjs+!@DSlG~&WnGY*Lq zjR&5C7i+#gSB!oGr^;#lJfok3`Z?CG?gFK+{i1%wnoEQ)DKL{YRnB;o&EihUWz0|1 zH=$1zxc-uaF4RR{simx$I}#4jx?RQbuqbRch1Yf9;Uw3oB_R#jFgB89TzxKRelU*$ zLZ&Ig+?vZ~)sZY03E<(qJ45k1qmgYUkKYrsIwD>M_^)O%x~rcC_Y+y@>A|Oo+}9dm zNS}(p$1j0!*3>aYp49PFwa{s9;ScceV delta 400 zcmX?k-~HuT_X+cPj7>}}ERzfqQpW7#>HvP~w$Gm;_?cBB(XpL63uF o`{cQTQb96n*3A4>ysqzC`&=e(pDDk)_vgvSFad+j-O1Mp0L@Q~NdN!<