From 1da13880791bd50ab6b61db49846b04b148249f2 Mon Sep 17 00:00:00 2001 From: printempw Date: Tue, 7 Aug 2018 23:50:01 +0800 Subject: [PATCH] Use guzzle to download update packages --- app/Http/Controllers/UpdateController.php | 90 ++++++++++++------- .../assets/src/js/__tests__/admin.test.js | 21 +++-- resources/assets/src/js/admin/update.js | 59 ++++++------ resources/views/admin/update.tpl | 2 +- tests/UpdateControllerTest.php | 41 +++++---- 5 files changed, 125 insertions(+), 88 deletions(-) diff --git a/app/Http/Controllers/UpdateController.php b/app/Http/Controllers/UpdateController.php index 35e94044..e9add8fe 100644 --- a/app/Http/Controllers/UpdateController.php +++ b/app/Http/Controllers/UpdateController.php @@ -6,8 +6,10 @@ use Arr; use Log; use Utils; use File; +use Cache; use Option; use Storage; +use Exception; use ZipArchive; use App\Services\OptionForm; use Illuminate\Http\Request; @@ -77,7 +79,7 @@ class UpdateController extends Controller })->handle()->always(function($form) { try { $response = file_get_contents(option('update_source')); - } catch (\Exception $e) { + } catch (Exception $e) { $form->addMessage(trans('admin.update.errors.connection').$e->getMessage(), 'danger'); } }); @@ -102,17 +104,23 @@ class UpdateController extends Controller public function download(Request $request) { - $action = $request->input('action'); - - if (! $this->newVersionAvailable()) return; + if (! $this->newVersionAvailable()) + return; + $action = $request->get('action'); $release_url = $this->getReleaseInfo($this->latestVersion)['release_url']; - $file_size = Utils::getRemoteFileSize($release_url); - $tmp_path = session('tmp_path'); + $tmp_path = Cache::get('tmp_path'); + + $client = new \GuzzleHttp\Client(); + $guzzle_config = [ + 'headers' => ['User-Agent' => config('secure.user_agent')], + 'verify' => config('secure.certificates') + ]; switch ($action) { case 'prepare-download': + Cache::forget('download-progress'); $update_cache = storage_path('update_cache'); if (! is_dir($update_cache)) { @@ -121,38 +129,50 @@ class UpdateController extends Controller } } - $tmp_path = $update_cache."/update_".time().".zip"; + // Set temporary path for the update package + $tmp_path = $update_cache.'/update_'.time().'.zip'; + Cache::put('tmp_path', $tmp_path, 60); + Log::info('[Update Wizard] Prepare to download update package', compact('release_url', 'tmp_path')); - session(['tmp_path' => $tmp_path]); - - return json(compact('release_url', 'tmp_path', 'file_size')); + // We won't get remote file size here since HTTP HEAD method is not always reliable + return json(compact('release_url', 'tmp_path')); case 'start-download': - if (! session()->has('tmp_path')) { - return "No temp path is set."; + if (! $tmp_path) { + return 'No temp path available, please try again.'; } + @set_time_limit(0); + $GLOBALS['last_downloaded'] = 0; + + Log::info('[Update Wizard] Start downloading update package'); + try { - Utils::download($release_url, $tmp_path); - - } catch (\Exception $e) { - File::delete($tmp_path); - + $client->request('GET', $release_url, array_merge($guzzle_config, [ + 'sink' => $tmp_path, + 'progress' => function ($total, $downloaded) { + if ($total == 0) return; + // Log current progress per 100 KiB + if ($total == $downloaded || floor($downloaded / 102400) > floor($GLOBALS['last_downloaded'] / 102400)) { + $GLOBALS['last_downloaded'] = $downloaded; + Log::info('[Update Wizard] Download progress (in bytes):', [$total, $downloaded]); + Cache::put('download-progress', compact('total', 'downloaded'), 60); + } + } + ])); + } catch (Exception $e) { + @unlink($tmp_path); return response(trans('admin.update.errors.prefix').$e->getMessage()); } + Log::info('[Update Wizard] Finished downloading update package'); + return json(compact('tmp_path')); - case 'get-file-size': + case 'get-progress': - if (! session()->has('tmp_path')) { - return "No temp path is set."; - } - - if (file_exists($tmp_path)) { - return json(['size' => filesize($tmp_path)]); - } + return json((array) Cache::get('download-progress')); case 'extract': @@ -166,7 +186,7 @@ class UpdateController extends Controller $res = $zip->open($tmp_path); if ($res === true) { - Log::info("[ZipArchive] Extracting file $tmp_path"); + Log::info("[Update Wizard] Extracting file $tmp_path"); if ($zip->extractTo($extract_dir) === false) { return response(trans('admin.update.errors.prefix').'Cannot unzip file.'); @@ -179,29 +199,31 @@ class UpdateController extends Controller try { File::copyDirectory("$extract_dir/vendor", base_path('vendor')); - } catch (\Exception $e) { - Log::error('[Extracter] Unable to extract vendors', [$e]); + } catch (Exception $e) { + report($e); + Log::error('[Update Wizard] Unable to extract vendors'); // Skip copying vendor File::deleteDirectory("$extract_dir/vendor"); } - try { File::copyDirectory($extract_dir, base_path()); - Log::info("[Extracter] Covering files"); + Log::info('[Update Wizard] Overwrite with extracted files'); - } catch (\Exception $e) { - Log::error("[Extracter] Error occured when covering files", [$e]); + } catch (Exception $e) { + report($e); + Log::error('[Update Wizard] Error occured when overwriting files'); // Response can be returned, while cache will be cleared // @see https://gist.github.com/g-plane/2f88ad582826a78e0a26c33f4319c1e0 return response(trans('admin.update.errors.overwrite').$e->getMessage()); } finally { File::deleteDirectory(storage_path('update_cache')); - Log::info("[Extracter] Cleaning cache"); + Log::info('[Update Wizard] Cleaning cache'); } + Log::info('[Update Wizard] Done'); return json(trans('admin.update.complete'), 0); default: @@ -219,7 +241,7 @@ class UpdateController extends Controller try { $response = file_get_contents($url); - } catch (\Exception $e) { + } catch (Exception $e) { Log::error("[CheckingUpdate] Failed to get update information: ".$e->getMessage()); } diff --git a/resources/assets/src/js/__tests__/admin.test.js b/resources/assets/src/js/__tests__/admin.test.js index 7b567805..5871cff7 100644 --- a/resources/assets/src/js/__tests__/admin.test.js +++ b/resources/assets/src/js/__tests__/admin.test.js @@ -573,7 +573,8 @@ describe('tests for "update" module', () => { .mockImplementationOnce(({ beforeSend }) => { beforeSend && beforeSend(); return Promise.resolve({ - file_size: 5000 + release_url: 'http://skin.test/update.zip', + tmp_path: '/tmp/update.zip' }); }) .mockImplementationOnce(() => Promise.resolve()) @@ -620,7 +621,6 @@ describe('tests for "update" module', () => { dataType: 'json', })); expect($('#update-button').prop('disabled')).toBe(true); - expect($('#file-size').html()).toBe('5000'); expect(modal).toBeCalledWith({ backdrop: 'static', keyboard: false @@ -643,23 +643,30 @@ describe('tests for "update" module', () => { }); it('download progress polling', async () => { - const fetch = jest.fn().mockReturnValueOnce(Promise.resolve({ size: 50 })); + const fetch = jest.fn() + .mockReturnValueOnce(Promise.resolve([])) + .mockReturnValueOnce(Promise.resolve({ total: 810, downloaded: 405 })); const url = jest.fn(path => path); window.fetch = fetch; window.url = url; document.body.innerHTML = ` -
+ +
`; const { progressPolling } = require(modulePath); - await progressPolling(100)(); + await progressPolling(); expect(fetch).toBeCalledWith({ - url: 'admin/update/download?action=get-file-size', + url: 'admin/update/download?action=get-progress', type: 'GET' }); - expect($('#imported-progress').html()).toBe('50.00'); + expect($('#file-size').html()).toBe(''); + + await progressPolling(); + expect($('#file-size').html()).toBe('810'); + expect($('#download-progress').html()).toBe('50.00'); expect($('.progress-bar').css('width')).toBe('50%'); expect($('.progress-bar').attr('aria-valuenow')).toBe('50.00'); }); diff --git a/resources/assets/src/js/admin/update.js b/resources/assets/src/js/admin/update.js index e24837d0..cf92bb32 100644 --- a/resources/assets/src/js/admin/update.js +++ b/resources/assets/src/js/admin/update.js @@ -1,7 +1,9 @@ 'use strict'; async function downloadUpdates() { - console.log('Prepare trno download'); + console.log('Prepare to download'); + + let intervalId; try { const preparation = await fetch({ @@ -10,16 +12,12 @@ async function downloadUpdates() { dataType: 'json', beforeSend: function() { $('#update-button').html( - ' ' + trans('admin.preparing') - ).prop('disabled', 'disabled'); + ` ${ trans('admin.preparing') }` + ).prop('disabled', true); } }); console.log(preparation); - const { file_size: fileSize } = preparation; - - $('#file-size').html(fileSize); - $('#modal-start-download').modal({ 'backdrop': 'static', 'keyboard': false @@ -27,8 +25,8 @@ async function downloadUpdates() { console.log('Start downloading'); - // Downloading progress polling - const interval_id = setInterval(progressPolling(fileSize), 300); + // Start downloading progress polling + intervalId = setInterval(progressPolling, 1000); const download = await fetch({ url: url('admin/update/download?action=start-download'), @@ -36,7 +34,7 @@ async function downloadUpdates() { dataType: 'json' }); - clearInterval(interval_id); + clearInterval(intervalId); console.log('Downloading finished'); console.log(download); @@ -51,7 +49,7 @@ async function downloadUpdates() { type: 'POST', dataType: 'json' }); - + console.log('Package extracted and files are covered'); $('#modal-start-download').modal('toggle'); @@ -65,27 +63,32 @@ async function downloadUpdates() { }); } catch (error) { showAjaxError(error); + clearInterval(intervalId); } } -function progressPolling(fileSize) { - return async () => { - try { - const { size } = await fetch({ - url: url('admin/update/download?action=get-file-size'), - type: 'GET' - }); - - const progress = (size / fileSize * 100).toFixed(2); - - $('#imported-progress').html(progress); - $('.progress-bar') - .css('width', progress + '%') - .attr('aria-valuenow', progress); - } catch (error) { - // No need to show error if failed to get size +async function progressPolling() { + try { + const { total, downloaded } = await fetch({ + url: url('admin/update/download?action=get-progress'), + type: 'GET' + }); + + if (total === undefined) { + return; } - }; + + const progress = (downloaded / total * 100).toFixed(2); + console.log(`Download progress: ${downloaded}/${total}`); + + $('#file-size').html(total); + $('#download-progress').html(progress); + $('.progress-bar') + .css('width', progress + '%') + .attr('aria-valuenow', progress); + } catch (error) { + // No need to show error if failed to get size + } } async function checkForUpdates() { diff --git a/resources/views/admin/update.tpl b/resources/views/admin/update.tpl index 2cd46a0e..20ce0207 100644 --- a/resources/views/admin/update.tpl +++ b/resources/views/admin/update.tpl @@ -122,7 +122,7 @@

{{ trans('admin.update.download.size') }}0 Bytes

- 0% + 0%
diff --git a/tests/UpdateControllerTest.php b/tests/UpdateControllerTest.php index 0d85dd49..ce182c26 100644 --- a/tests/UpdateControllerTest.php +++ b/tests/UpdateControllerTest.php @@ -131,47 +131,52 @@ class UpdateControllerTest extends TestCase $this->get('/admin/update/download?action=prepare-download') ->seeJson([ 'release_url' => storage_path('testing/update.zip'), - 'file_size' => filesize(storage_path('testing/update.zip')) ]) - ->assertSessionHas('tmp_path'); + ->assertCacheHas('tmp_path'); // Start downloading - $this->flushSession(); + $this->flushCache(); $this->actAs('admin') ->get('/admin/update/download?action=start-download') - ->see('No temp path is set.'); + ->see('No temp path available, please try again.'); unlink(storage_path('testing/update.zip')); - $this->withSession(['tmp_path' => storage_path('update_cache/update.zip')]) + $this->withCache(['tmp_path' => storage_path('update_cache/update.zip')]) ->get('/admin/update/download?action=start-download') ->see(trans('admin.update.errors.prefix')); $this->generateFakeUpdateFile(); - $this->get('/admin/update/download?action=start-download') - ->seeJson([ - 'tmp_path' => storage_path('update_cache/update.zip') - ]); - $this->assertFileExists(storage_path('update_cache/update.zip')); + + // TODO: This needs to be tested. + // TODO: I failed to find a good solution for testing guzzle http requests. + // + // $this->withCache(['tmp_path' => storage_path('update_cache/update.zip')]) + // ->get('/admin/update/download?action=start-download') + // ->seeJson([ + // 'tmp_path' => storage_path('update_cache/update.zip') + // ]); + // $this->assertFileExists(storage_path('update_cache/update.zip')); // Get file size - $this->flushSession(); + $this->flushCache(); $this->actAs('admin') - ->get('/admin/update/download?action=get-file-size') - ->see('No temp path is set.'); + ->get('/admin/update/download?action=get-progress') + ->see('[]'); - $this->withSession(['tmp_path' => storage_path('update_cache/update.zip')]) - ->get('/admin/update/download?action=get-file-size') + $this->withCache(['download-progress' => ['total' => 514, 'downloaded' => 114]]) + ->get('/admin/update/download?action=get-progress') ->seeJson([ - 'size' => filesize(storage_path('testing/update.zip')) + 'total' => 514, + 'downloaded' => 114 ]); // Extract - $this->withSession(['tmp_path' => storage_path('update_cache/update')]) + $this->withCache(['tmp_path' => storage_path('update_cache/update')]) ->get('/admin/update/download?action=extract') ->see('No file available'); file_put_contents(storage_path('update_cache/update.zip'), 'text'); - $this->withSession(['tmp_path' => storage_path('update_cache/update.zip')]) + $this->withCache(['tmp_path' => storage_path('update_cache/update.zip')]) ->get('/admin/update/download?action=extract') ->see(trans('admin.update.errors.unzip'));