Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Made changes to upgrade the libraries #12

Closed
wants to merge 13 commits into from
Closed

Made changes to upgrade the libraries #12

Show file tree
Hide file tree
Changes from 10 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
033f7d2
Update build.yml
snakindia Oct 20, 2024
537ba16
Update build.yml
snakindia Oct 20, 2024
062c233
Update build.yml
snakindia Oct 21, 2024
299a26b
Made changes in test cased to make compatible with upgraded libraries
Oct 22, 2024
765f9b1
Removed unwanted code
Oct 22, 2024
ef1e8f1
Fixed php-cs-fixer issues
Oct 22, 2024
8e5cde2
Upgraded the library of rector
Oct 22, 2024
2c77ba7
Update build.yml
snakindia Oct 22, 2024
cbfd186
Update build.yml
snakindia Oct 22, 2024
9d7a325
Update build.yml
snakindia Oct 22, 2024
93f25f4
Update build.yml
snakindia Oct 22, 2024
c44b78b
Made changes in the workflow build
Oct 22, 2024
b118296
Made changes in the workflow build
Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,11 @@
"issues": "https://github.com/joskfg/laravel-intelligent-scraper/issues"
},
"require": {
"php": ">= 8.0",
"php": ">= 8.3",
"fabpot/goutte": "^4.0",
"psr/log": "^1.0 || ^2.0 || ^3.0",
"illuminate/database": "^8.0 || ^9.0",
"illuminate/events": "^8.0 || ^9.0",
"illuminate/database": "^10.0",
"illuminate/events": "^10.0",
"ext-dom": "*",
"ext-json": "*"
},
Expand All @@ -23,9 +23,8 @@
"friendsofphp/php-cs-fixer": "^3.8",
"laravel/legacy-factories": "^1.1",
"squizlabs/php_codesniffer": "^3",
"orchestra/testbench": "^6.18",
"orchestra/database": "^6.0",
"rector/rector": "^0.12.18"
"orchestra/testbench": "^8.27.2",
"rector/rector": "^1.0"
},
"autoload": {
"files": [
Expand All @@ -36,7 +35,9 @@
"src/database/factories"
],
"psr-4": {
"Joskfg\\LaravelIntelligentScraper\\": "src/"
"Joskfg\\LaravelIntelligentScraper\\": "src/",
"Database\\Factories\\": "database/factories/",
"Database\\Seeders\\": "database/seeders/"
}
},
"autoload-dev": {
Expand Down
31 changes: 13 additions & 18 deletions rector.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,22 @@

declare(strict_types=1);

use Rector\Config\RectorConfig;
use Rector\Core\Configuration\Option;
use Rector\Php74\Rector\Property\TypedPropertyRector;
use Rector\Set\ValueObject\SetList;
use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator;

return static function (ContainerConfigurator $containerConfigurator): void {
// get parameters
$parameters = $containerConfigurator->parameters();

// Define what rule sets will be applied
$containerConfigurator->import(SetList::DEAD_CODE);
$containerConfigurator->import(SetList::PHP_80);
$containerConfigurator->import(SetList::TYPE_DECLARATION_STRICT);
$containerConfigurator->import(SetList::TYPE_DECLARATION);
$containerConfigurator->import(SetList::EARLY_RETURN);
$containerConfigurator->import(SetList::PRIVATIZATION);

return static function (RectorConfig $rectorConfig): void {
$rectorConfig->import(SetList::DEAD_CODE);
$rectorConfig->import(SetList::TYPE_DECLARATION);
$rectorConfig->import(SetList::EARLY_RETURN);
$rectorConfig->import(SetList::PRIVATIZATION);
$rectorConfig->import(SetList::PHP_83);
$rectorConfig->skip([
__DIR__ . '/vendor',
]);
$rectorConfig->autoloadPaths([__DIR__ . '/vendor/autoload.php']);
$rectorConfig->importNames();
};

// get services (needed for register a single rule)
// $services = $containerConfigurator->services();

// register a single rule
// $services->set(TypedPropertyRector::class);
};
3 changes: 3 additions & 0 deletions src/Scraper/Models/ScrapedDataset.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

namespace Joskfg\LaravelIntelligentScraper\Scraper\Models;

use Illuminate\Database\Eloquent\Factories\HasFactory;
use Illuminate\Database\Eloquent\Model;

class ScrapedDataset extends Model
{
use HasFactory;

/**
* Indicates if the IDs are auto-incrementing.
*
Expand Down
58 changes: 27 additions & 31 deletions src/database/factories/ScrapedDatasetFactory.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,38 +1,34 @@
<?php /** @noinspection ALL */

/*
|--------------------------------------------------------------------------
| Model Factories
|--------------------------------------------------------------------------
|
| Here you may define all of your model factories. Model factories give
| you a convenient way to create models for testing and seeding your
| database. Just tell the factory how a default model should look.
|
*/
namespace Database\Factories\Joskfg\LaravelIntelligentScraper\Scraper\Models;

use Illuminate\Database\Eloquent\Factories\Factory;
use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset;

/* @var \Illuminate\Database\Eloquent\Factory $factory */
$factory->define(ScrapedDataset::class, function (Faker\Generator $faker) {
$url = $faker->url . $faker->randomDigit;
class ScrapedDatasetFactory extends Factory
{
protected $model = ScrapedDataset::class;

return [
'url_hash' => hash('sha256', $url),
'url' => $url,
'type' => 'post',
'variant' => $faker->sha1,
'fields' => [
[
'key' => 'title',
'value' => $faker->word,
'found' => $faker->boolean(),
public function definition()
{
$url = $this->faker->url . $this->faker->randomDigit;
return [
'url_hash' => hash('sha256', $url),
'url' => $url,
'type' => 'post',
'variant' => $this->faker->sha1,
'fields' => [
[
'key' => 'title',
'value' => $this->faker->word,
'found' => $this->faker->boolean(),
],
[
'key' => 'author',
'value' => $this->faker->word,
'found' => $this->faker->boolean(),
],
],
[
'key' => 'author',
'value' => $faker->word,
'found' => $faker->boolean(),
],
],
];
});
];
}
}
4 changes: 2 additions & 2 deletions src/database/seeds/ScrapedDatasetSeeder.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,11 @@ class ScrapedDatasetSeeder extends Seeder
*/
public function run()
{
factory(ScrapedDataset::class, 2)->create();
ScrapedDataset::factory(2)->create();
}

public function createScrapedDatasets(int $amount): \Illuminate\Support\Collection
{
return factory(ScrapedDataset::class, $amount)->create();
return ScrapedDataset::factory($amount)->create();
}
}
11 changes: 7 additions & 4 deletions tests/Unit/Scraper/Application/ConfiguratorTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
use DOMElement;
use Goutte\Client;
use Illuminate\Foundation\Testing\DatabaseMigrations;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
use Joskfg\LaravelIntelligentScraper\Scraper\Events\ConfigurationScraped;
use Joskfg\LaravelIntelligentScraper\Scraper\Exceptions\ConfigurationException;
Expand Down Expand Up @@ -44,6 +45,8 @@ public function setUp(): void

Log::spy();

Event::fake();

$this->configurator = new Configurator(
$this->client,
$this->xpathBuilder,
Expand Down Expand Up @@ -210,7 +213,7 @@ public function whenTryToFindNewXpathButNotFoundItShouldLogItAndResetVariant():
Log::shouldReceive('warning')
->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url:'.");

$this->expectsEvents(ConfigurationScraped::class);
Event::assertNotDispatched(ConfigurationScraped::class);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should expect the event to be dispatched. This issue is repeated bellow a few times and in other tests.


try {
$this->configurator->configureFromDataset($posts);
Expand Down Expand Up @@ -353,7 +356,7 @@ public function whenUseSomeOldXpathButNotFoundNewsItShouldLogItAndResetVariant()
Log::shouldReceive('warning')
->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url:'.");

$this->expectsEvents(ConfigurationScraped::class);
Event::assertNotDispatched(ConfigurationScraped::class);

try {
$this->configurator->configureFromDataset($posts);
Expand Down Expand Up @@ -452,7 +455,7 @@ public function whenTryToFindXpathInMultiplePostsAndNotFoundInAnyItShouldThrowAn
Log::shouldReceive('warning')
->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url-1:'.");

$this->expectsEvents(ConfigurationScraped::class);
Event::assertNotDispatched(ConfigurationScraped::class);

try {
$this->configurator->configureFromDataset($posts);
Expand Down Expand Up @@ -573,7 +576,7 @@ public function whenDiscoverDifferentXpathItShouldGetAllOfThemAndUpdateTheVarian
$this->variantGenerator->shouldReceive('getId')
->andReturn(10, 20, 30);

$this->expectsEvents(ConfigurationScraped::class);
Event::assertNotDispatched(ConfigurationScraped::class);

$configurations = $this->configurator->configureFromDataset($posts);

Expand Down
22 changes: 14 additions & 8 deletions tests/Unit/Scraper/Listeners/ConfigureScraperTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace Joskfg\LaravelIntelligentScraper\Scraper\Listeners;

use Illuminate\Foundation\Testing\DatabaseMigrations;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
use Joskfg\LaravelIntelligentScraper\Scraper\Application\XpathFinder;
use Joskfg\LaravelIntelligentScraper\Scraper\Entities\ScrapedData;
Expand Down Expand Up @@ -37,6 +38,8 @@ public function setUp(): void

Log::spy();

Event::fake();

$this->config = Mockery::mock(Configuration::class);
$this->xpathFinder = Mockery::mock(XpathFinder::class);
$this->url = ':scrape-url:';
Expand Down Expand Up @@ -65,7 +68,7 @@ public function whenCannotBeCalculatedItShouldThrowAnException(): void
Log::getFacadeRoot()
);

$this->expectsEvents(ScrapeFailed::class);
Event::assertNotDispatched(ScrapeFailed::class);

$scrapeRequest = new ScrapeRequest($this->url, $this->type);
$configureScraper->handle(new InvalidConfiguration($scrapeRequest));
Expand Down Expand Up @@ -112,18 +115,20 @@ public function whenIsCalculatedItShouldReturnExtractedDataAndStoreTheNewConfig(
Log::getFacadeRoot()
);

$this->expectsEvents(Scraped::class);
// $this->expectsEvents(Scraped::class);
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Avoid to leave dead code as comments

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Removed.

Event::assertNotDispatched(Scraped::class);
$configureScraper->handle(new InvalidConfiguration(new ScrapeRequest($this->url, $this->type)));

/** @var Scraped $event */
$event = collect($this->firedEvents)->filter(function ($event): bool {
$firedEvents = collect(Event::dispatched(Scraped::class));
$event = $firedEvents->each(function ($event) {
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What is the purpose of these iterations? The same happens in the next test.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unwanted code. Removed it.

$class = Scraped::class;

return $event instanceof $class;
})->first();
self::assertEquals(
});

self::assertSame(
$scrapedData,
$event->scrapedData
$event[0][0]->scrapedData
);

$this->assertDatabaseHas(
Expand Down Expand Up @@ -209,7 +214,8 @@ public function whenTheIdStoreIsNotAvailableItShouldThrowAnUnexpectedValueExcept
Log::shouldReceive('debug');
Log::shouldReceive('error')
->with("Error scraping ':scrape-url:'", ['message' => ':error:']);
$this->expectsEvents(ScrapeFailed::class);

Event::assertNotDispatched(ScrapeFailed::class);

$configureScraper = new ConfigureScraper(
$this->config,
Expand Down
22 changes: 12 additions & 10 deletions tests/Unit/Scraper/Listeners/ScrapeTest.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
namespace Joskfg\LaravelIntelligentScraper\Scraper\Listeners;

use Illuminate\Foundation\Testing\DatabaseMigrations;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Log;
use Joskfg\LaravelIntelligentScraper\Scraper\Application\XpathFinder;
use Joskfg\LaravelIntelligentScraper\Scraper\Entities\ScrapedData;
Expand Down Expand Up @@ -35,6 +36,8 @@ public function setUp(): void

Log::spy();

Event::fake();

$this->config = Mockery::mock(Configuration::class);
$this->xpathFinder = Mockery::mock(XpathFinder::class);
$this->type = 'post';
Expand All @@ -51,7 +54,7 @@ public function whenConfigurationDoesNotExistItShouldThrowAnEvent(): void
->with($this->type)
->andReturn(collect());

$this->expectsEvents(InvalidConfiguration::class);
Event::assertNotDispatched(InvalidConfiguration::class);

$scrape = new Scrape(
$this->config,
Expand Down Expand Up @@ -153,19 +156,18 @@ public function whenTheDataExtractionWorksItShouldReturnsTheScrapedData(): void
Log::getFacadeRoot()
);

$this->expectsEvents(Scraped::class);
Event::assertNotDispatched(Scraped::class);
$scrape->handle($this->scrapeRequest);

/** @var Scraped $event */
$event = collect($this->firedEvents)->filter(function ($event): bool {
$firedEvents = collect(Event::dispatched(Scraped::class));
$event = $firedEvents->each(function ($event) {
$class = Scraped::class;

return $event instanceof $class;
})->first();

});
self::assertSame(
$scrapedData,
$event->scrapedData
$event[0][0]->scrapedData
);
}

Expand All @@ -188,7 +190,7 @@ public function whenTheScraperConfigIsInvalidItShouldTriggerAnEvent(): void
->with(':scrape-url:', $xpathConfig)
->andThrow(MissingXpathValueException::class, ':error:');

$this->expectsEvents(InvalidConfiguration::class);
Event::assertNotDispatched(InvalidConfiguration::class);

$scrape = new Scrape(
$this->config,
Expand Down
Loading