Browse Source

TASK: Add recover file

feature/kickstart
Daniel Siepmann 4 years ago
parent
commit
09bdc7e73f
Signed by: Daniel Siepmann
GPG Key ID: 33D6629915560EF4
  1. 7
      comparison
  2. 48
      src/Command/CompareCommand.php
  3. 52
      src/Command/CreateBaseCommand.php
  4. 37
      src/Model/UrlListDto.php
  5. 48
      src/Model/UrlListDtoFactory.php
  6. 5
      src/Service/Screenshot/CrawlerService.php
  7. 5
      src/Service/Screenshot/Service.php

7
comparison

@ -1,6 +1,12 @@
#!/usr/bin/env php
<?php
declare(ticks = 1);
pcntl_signal(SIGINT, function () {
exit(99);
});
require __DIR__ . '/vendor/autoload.php';
use Codappix\WebsiteComparison\Command\CompareCommand;
@ -25,6 +31,7 @@ $chromeDriver = (function () {
return ChromeDriver::start(null, $chromeDriverService);
})();
$application = new Application();
$application->setDispatcher($eventDispatcher);

48
src/Command/CompareCommand.php

@ -21,6 +21,8 @@ namespace Codappix\WebsiteComparison\Command;
* 02110-1301, USA.
*/
use Codappix\WebsiteComparison\Model\UrlListDto;
use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
use Codappix\WebsiteComparison\Service\Screenshot\CompareService;
use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
use Codappix\WebsiteComparison\Service\Screenshot\Service;
@ -90,6 +92,13 @@ class CompareCommand extends Command
'The width for screen resolution and screenshots.',
3840
)
->addOption(
'recoverFile',
null,
InputOption::VALUE_OPTIONAL,
'Path to json-File with state of stopped process, used to recover process.',
''
)
->addArgument(
'baseUrl',
@ -101,6 +110,7 @@ class CompareCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output)
{
$baseUrl = $input->getArgument('baseUrl');
$screenshotService = new Service(
$this->eventDispatcher,
@ -111,7 +121,7 @@ class CompareCommand extends Command
$screenshotCrawler = new CrawlerService(
$this->webDriver,
$screenshotService,
$input->getArgument('baseUrl')
$baseUrl
);
$compareService = new CompareService(
@ -120,15 +130,47 @@ class CompareCommand extends Command
$input->getOption('screenshotDir'),
$input->getOption('diffResultDir')
);
$this->registerEvents($output, $compareService);
$screenshotCrawler->crawl();
$linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
$this->registerEvents($output, $compareService);
try {
$screenshotCrawler->crawl($linkList);
} catch (\Exception $e) {
file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
$output->writeln(sprintf(
'<comment>Saved current state for recovering in "%s".</comment>',
$this->getJsonFilePath($screenshotService, $baseUrl)
));
throw $e;
}
if ($compareService->hasDifferences()) {
return 255;
}
}
protected function getLinkList(
string $baseUrl,
string $recoverFile = ''
): UrlListDto {
$factory = new UrlListDtoFactory();
if (trim($recoverFile) !== '') {
return $factory->createWithByConfigurationFile($recoverFile);
}
return $factory->createWithBaseUrl($baseUrl);
}
protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
{
return $screenshotService->getScreenshotDir() .
DIRECTORY_SEPARATOR .
$screenshotService->getScreenshotTarget($baseUrl, 'json')
;
}
protected function registerEvents(OutputInterface $output, CompareService $compareService)
{
$this->eventDispatcher->addListener(

52
src/Command/CreateBaseCommand.php

@ -21,6 +21,8 @@ namespace Codappix\WebsiteComparison\Command;
* 02110-1301, USA.
*/
use Codappix\WebsiteComparison\Model\UrlListDto;
use Codappix\WebsiteComparison\Model\UrlListDtoFactory;
use Codappix\WebsiteComparison\Service\Screenshot\CrawlerService;
use Codappix\WebsiteComparison\Service\Screenshot\Service;
use Facebook\WebDriver\Remote\RemoteWebDriver;
@ -75,6 +77,13 @@ class CreateBaseCommand extends Command
'The width for screen resolution and screenshots.',
3840
)
->addOption(
'recoverFile',
null,
InputOption::VALUE_OPTIONAL,
'Path to json-File with state of stopped process, used to recover process.',
''
)
->addArgument(
'baseUrl',
@ -86,20 +95,55 @@ class CreateBaseCommand extends Command
protected function execute(InputInterface $input, OutputInterface $output)
{
$this->registerEvents($output);
$baseUrl = $input->getArgument('baseUrl');
$screenshotDir = $input->getOption('screenshotDir');
$screenshotService = new Service(
$this->eventDispatcher,
$input->getOption('screenshotDir'),
$screenshotDir,
$input->getOption('screenshotWidth')
);
$screenshotCrawler = new CrawlerService(
$this->webDriver,
$screenshotService,
$input->getArgument('baseUrl')
$baseUrl
);
$screenshotCrawler->crawl();
$linkList = $this->getLinkList($baseUrl, $input->getOption('recoverFile'));
$this->registerEvents($output);
try {
$screenshotCrawler->crawl($linkList);
} catch (\Exception $e) {
file_put_contents($this->getJsonFilePath($screenshotService, $baseUrl), json_encode($linkList));
$output->writeln(sprintf(
'<comment>Saved current state for recovering in "%s".</comment>',
$this->getJsonFilePath($screenshotService, $baseUrl)
));
throw $e;
}
}
protected function getLinkList(
string $baseUrl,
string $recoverFile = ''
): UrlListDto {
$factory = new UrlListDtoFactory();
if (trim($recoverFile) !== '') {
return $factory->createWithByConfigurationFile($recoverFile);
}
return $factory->createWithBaseUrl($baseUrl);
}
protected function getJsonFilePath(Service $screenshotService, string $baseUrl): string
{
return $screenshotService->getScreenshotDir() .
DIRECTORY_SEPARATOR .
$screenshotService->getScreenshotTarget($baseUrl, 'json')
;
}
protected function registerEvents(OutputInterface $output)

37
src/Model/UrlListDto.php

@ -22,21 +22,32 @@ namespace Codappix\WebsiteComparison\Model;
*/
/**
* List of urls with two states.
*
* Allows to have a single queue of urls to work on.
*/
class UrlListDto
class UrlListDto implements \JsonSerializable
{
protected $finishedUrls = [];
protected $upcomingUrls = [];
public function addUrl(string $link)
public function addUrl(string $url)
{
if ($this->isUrlKnown($link)) {
if ($this->isUrlKnown($url)) {
return;
}
$this->upcomingUrls[] = $link;
$this->upcomingUrls[] = $url;
}
public function addFinishedUrl(string $url)
{
if ($this->isUrlKnown($url)) {
return;
}
$this->finishedUrls[] = $url;
}
public function getNextUrl(): string
@ -44,17 +55,25 @@ class UrlListDto
return reset($this->upcomingUrls) ?? '';
}
public function markUrlAsFinished(string $link)
public function markUrlAsFinished(string $url)
{
$upcomingEntry = array_search($link, $this->upcomingUrls);
$upcomingEntry = array_search($url, $this->upcomingUrls);
unset($this->upcomingUrls[$upcomingEntry]);
$this->finishedUrls[] = $link;
$this->finishedUrls[] = $url;
}
public function isUrlKnown(string $url): bool
{
return in_array($url, $this->finishedUrls) || in_array($url, $this->upcomingUrls);
}
public function isUrlKnown(string $link): bool
public function jsonSerialize()
{
return in_array($link, $this->finishedUrls) || in_array($link, $this->upcomingUrls);
return [
'finishedUrls' => $this->finishedUrls,
'upcomingUrls' => $this->upcomingUrls,
];
}
}

48
src/Model/UrlListDtoFactory.php

@ -0,0 +1,48 @@
<?php
namespace Codappix\WebsiteComparison\Model;
/*
* Copyright (C) 2018 Daniel Siepmann <coding@daniel-siepmann.de>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
* 02110-1301, USA.
*/
class UrlListDtoFactory
{
public function createWithBaseUrl(string $baseUrl): UrlListDto
{
$urlList = new UrlListDto();
$urlList->addUrl($baseUrl);
return $urlList;
}
public function createWithByConfigurationFile(string $configFile): UrlListDto
{
$urlList = new UrlListDto();
$config = json_decode(file_get_contents($configFile), true);
foreach ($config['upcomingUrls'] as $url) {
$urlList->addUrl($url);
}
foreach ($config['finishedUrls'] as $url) {
$urlList->addFinishedUrl($url);
}
return $urlList;
}
}

5
src/Service/Screenshot/CrawlerService.php

@ -54,11 +54,8 @@ class CrawlerService
$this->baseUrl = rtrim($baseUrl, '/') . '/';
}
public function crawl()
public function crawl(UrlListDto $linkList)
{
$linkList = new UrlListDto();
$linkList->addUrl($this->baseUrl);
while ($url = $linkList->getNextUrl()) {
$this->driver->get($url);
$screenshotHeight = $this->driver->findElement(WebDriverBy::cssSelector('body'))

5
src/Service/Screenshot/Service.php

@ -69,6 +69,7 @@ class Service
'--screenshot=' . $completeScreenshotTarget,
$url
]);
$screenshotProcess->setTimeout(60 * 2);
// TODO: Check for success
$screenshotProcess->run();
@ -83,7 +84,7 @@ class Service
return $completeScreenshotTarget;
}
protected function getScreenshotTarget(string $url): string
public function getScreenshotTarget(string $url, string $suffix = 'png'): string
{
$uri = new Uri($url);
@ -100,7 +101,7 @@ class Service
return trim($string, ' /') !== '';
}
)
) . '.png';
) . '.' . $suffix;
}
public function getScreenshotDir(): string

Loading…
Cancel
Save