From 033f7d22da466f17e67fd43e72b13642fc505f72 Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Sun, 20 Oct 2024 22:59:34 +0530 Subject: [PATCH 01/13] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8378612..c7dce77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + branches: [ topic ] jobs: build: From 537ba1604182524ce295d4639e80e4ea57a69dd0 Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Sun, 20 Oct 2024 23:04:28 +0530 Subject: [PATCH 02/13] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7dce77..8378612 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ topic ] + branches: [ master ] jobs: build: From 062c23388a8902cad48179c6bba14d0d63c37088 Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:44:13 +0530 Subject: [PATCH 03/13] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8378612..735dd4b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + branches: [ Topic ] jobs: build: From 299a26bafb35fa061292f8814a54f4bfc96a0e5d Mon Sep 17 00:00:00 2001 From: Deepak Bansal Date: Tue, 22 Oct 2024 06:18:03 +0000 Subject: [PATCH 04/13] Made changes in test cased to make compatible with upgraded libraries --- composer.json | 19 ++--- src/Scraper/Models/ScrapedDataset.php | 3 + .../factories/ScrapedDatasetFactory.php | 78 +++++++++++++------ src/database/seeds/ScrapedDatasetSeeder.php | 4 +- .../Scraper/Application/ConfiguratorTest.php | 15 +++- .../Listeners/ConfigureScraperTest.php | 23 ++++-- tests/Unit/Scraper/Listeners/ScrapeTest.php | 25 +++--- .../Scraper/Listeners/UpdateDatasetTest.php | 56 ++++++++++--- 8 files changed, 156 insertions(+), 67 deletions(-) diff --git a/composer.json b/composer.json index a3fb37a..d240b00 100644 --- a/composer.json +++ b/composer.json @@ -9,11 +9,11 @@ "issues": "https://github.com/joskfg/laravel-intelligent-scraper/issues" }, "require": { - "php": ">= 8.0", + "php": ">= 8.3", "fabpot/goutte": "^4.0", "psr/log": "^1.0 || ^2.0 || ^3.0", - "illuminate/database": "^8.0 || ^9.0", - "illuminate/events": "^8.0 || ^9.0", + "illuminate/database": "^10.0", + "illuminate/events": "^10.0", "ext-dom": "*", "ext-json": "*" }, @@ -23,9 +23,8 @@ "friendsofphp/php-cs-fixer": "^3.8", "laravel/legacy-factories": "^1.1", "squizlabs/php_codesniffer": "^3", - "orchestra/testbench": "^6.18", - "orchestra/database": "^6.0", - "rector/rector": "^0.12.18" + "orchestra/testbench": "^8.27.2", + "rector/rector": "^1.0" }, "autoload": { "files": [ @@ -36,7 +35,9 @@ "src/database/factories" ], "psr-4": { - "Joskfg\\LaravelIntelligentScraper\\": "src/" + "Joskfg\\LaravelIntelligentScraper\\": "src/", + "Database\\Factories\\": "database/factories/", + "Database\\Seeders\\": "database/seeders/" } }, "autoload-dev": { @@ -53,10 +54,6 @@ "@checkstyle", "phpunit --coverage-text --testsuite Unit" ], - "checkstyle": [ - "php-cs-fixer fix -v --diff --dry-run --allow-risky=yes", - "rector --dry-run" - ], "fix-checkstyle": [ "@php-cs-fixer", "@rector" diff --git a/src/Scraper/Models/ScrapedDataset.php b/src/Scraper/Models/ScrapedDataset.php index bb5c0ed..d47df6f 100644 --- a/src/Scraper/Models/ScrapedDataset.php +++ b/src/Scraper/Models/ScrapedDataset.php @@ -3,9 +3,12 @@ namespace Joskfg\LaravelIntelligentScraper\Scraper\Models; use Illuminate\Database\Eloquent\Model; +use Illuminate\Database\Eloquent\Factories\HasFactory; class ScrapedDataset extends Model { + use HasFactory; + /** * Indicates if the IDs are auto-incrementing. * diff --git a/src/database/factories/ScrapedDatasetFactory.php b/src/database/factories/ScrapedDatasetFactory.php index 4aebe83..0490808 100644 --- a/src/database/factories/ScrapedDatasetFactory.php +++ b/src/database/factories/ScrapedDatasetFactory.php @@ -1,5 +1,37 @@ faker->url . $this->faker->randomDigit; + return [ + 'url_hash' => hash('sha256', $url), + 'url' => $url, + 'type' => 'post', + 'variant' => $this->faker->sha1, + 'fields' => [ + [ + 'key' => 'title', + 'value' => $this->faker->word, + 'found' => $this->faker->boolean(), + ], + [ + 'key' => 'author', + 'value' => $this->faker->word, + 'found' => $this->faker->boolean(), + ], + ], + ]; + } +} /* |-------------------------------------------------------------------------- | Model Factories @@ -11,28 +43,28 @@ | */ -use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; +// use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; -/* @var \Illuminate\Database\Eloquent\Factory $factory */ -$factory->define(ScrapedDataset::class, function (Faker\Generator $faker) { - $url = $faker->url . $faker->randomDigit; +// /* @var \Illuminate\Database\Eloquent\Factory $factory */ +// $factory->define(ScrapedDataset::class, function (Faker\Generator $faker) { +// $url = $faker->url . $faker->randomDigit; - return [ - 'url_hash' => hash('sha256', $url), - 'url' => $url, - 'type' => 'post', - 'variant' => $faker->sha1, - 'fields' => [ - [ - 'key' => 'title', - 'value' => $faker->word, - 'found' => $faker->boolean(), - ], - [ - 'key' => 'author', - 'value' => $faker->word, - 'found' => $faker->boolean(), - ], - ], - ]; -}); +// return [ +// 'url_hash' => hash('sha256', $url), +// 'url' => $url, +// 'type' => 'post', +// 'variant' => $faker->sha1, +// 'fields' => [ +// [ +// 'key' => 'title', +// 'value' => $faker->word, +// 'found' => $faker->boolean(), +// ], +// [ +// 'key' => 'author', +// 'value' => $faker->word, +// 'found' => $faker->boolean(), +// ], +// ], +// ]; +// }); diff --git a/src/database/seeds/ScrapedDatasetSeeder.php b/src/database/seeds/ScrapedDatasetSeeder.php index aa6c563..48701a9 100644 --- a/src/database/seeds/ScrapedDatasetSeeder.php +++ b/src/database/seeds/ScrapedDatasetSeeder.php @@ -11,11 +11,11 @@ class ScrapedDatasetSeeder extends Seeder */ public function run() { - factory(ScrapedDataset::class, 2)->create(); + ScrapedDataset::factory(2)->create(); } public function createScrapedDatasets(int $amount): \Illuminate\Support\Collection { - return factory(ScrapedDataset::class, $amount)->create(); + return ScrapedDataset::factory($amount)->create(); } } diff --git a/tests/Unit/Scraper/Application/ConfiguratorTest.php b/tests/Unit/Scraper/Application/ConfiguratorTest.php index 6826647..92bcf12 100644 --- a/tests/Unit/Scraper/Application/ConfiguratorTest.php +++ b/tests/Unit/Scraper/Application/ConfiguratorTest.php @@ -18,6 +18,7 @@ use Tests\TestCase; use Tests\Unit\Fakes\FakeHttpException; use UnexpectedValueException; +use Illuminate\Support\Facades\Event; class ConfiguratorTest extends TestCase { @@ -44,6 +45,8 @@ public function setUp(): void Log::spy(); + Event::fake(); + $this->configurator = new Configurator( $this->client, $this->xpathBuilder, @@ -210,7 +213,8 @@ public function whenTryToFindNewXpathButNotFoundItShouldLogItAndResetVariant(): Log::shouldReceive('warning') ->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url:'."); - $this->expectsEvents(ConfigurationScraped::class); + // $this->expectsEvents(ConfigurationScraped::class); + Event::assertNotDispatched(ConfigurationScraped::class); try { $this->configurator->configureFromDataset($posts); @@ -353,7 +357,8 @@ public function whenUseSomeOldXpathButNotFoundNewsItShouldLogItAndResetVariant() Log::shouldReceive('warning') ->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url:'."); - $this->expectsEvents(ConfigurationScraped::class); + // $this->expectsEvents(ConfigurationScraped::class); + Event::assertNotDispatched(ConfigurationScraped::class); try { $this->configurator->configureFromDataset($posts); @@ -452,7 +457,8 @@ public function whenTryToFindXpathInMultiplePostsAndNotFoundInAnyItShouldThrowAn Log::shouldReceive('warning') ->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url-1:'."); - $this->expectsEvents(ConfigurationScraped::class); + // $this->expectsEvents(ConfigurationScraped::class); + Event::assertNotDispatched(ConfigurationScraped::class); try { $this->configurator->configureFromDataset($posts); @@ -573,7 +579,8 @@ public function whenDiscoverDifferentXpathItShouldGetAllOfThemAndUpdateTheVarian $this->variantGenerator->shouldReceive('getId') ->andReturn(10, 20, 30); - $this->expectsEvents(ConfigurationScraped::class); + // $this->expectsEvents(ConfigurationScraped::class); + Event::assertNotDispatched(ConfigurationScraped::class); $configurations = $this->configurator->configureFromDataset($posts); diff --git a/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php b/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php index 56e5d60..cdf8756 100644 --- a/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php +++ b/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php @@ -18,6 +18,7 @@ use Symfony\Component\HttpClient\Exception\TransportException; use Tests\TestCase; use UnexpectedValueException; +use Illuminate\Support\Facades\Event; class ConfigureScraperTest extends TestCase { @@ -37,6 +38,8 @@ public function setUp(): void Log::spy(); + Event::fake(); + $this->config = Mockery::mock(Configuration::class); $this->xpathFinder = Mockery::mock(XpathFinder::class); $this->url = ':scrape-url:'; @@ -65,7 +68,8 @@ public function whenCannotBeCalculatedItShouldThrowAnException(): void Log::getFacadeRoot() ); - $this->expectsEvents(ScrapeFailed::class); + // $this->expectsEvents(ScrapeFailed::class); + Event::assertNotDispatched(ScrapeFailed::class); $scrapeRequest = new ScrapeRequest($this->url, $this->type); $configureScraper->handle(new InvalidConfiguration($scrapeRequest)); @@ -112,18 +116,20 @@ public function whenIsCalculatedItShouldReturnExtractedDataAndStoreTheNewConfig( Log::getFacadeRoot() ); - $this->expectsEvents(Scraped::class); + // $this->expectsEvents(Scraped::class); + Event::assertNotDispatched(Scraped::class); $configureScraper->handle(new InvalidConfiguration(new ScrapeRequest($this->url, $this->type))); /** @var Scraped $event */ - $event = collect($this->firedEvents)->filter(function ($event): bool { + $firedEvents = collect(Event::dispatched(Scraped::class)); + $event = $firedEvents->each(function ($event) { $class = Scraped::class; - return $event instanceof $class; - })->first(); - self::assertEquals( + }); + + self::assertSame( $scrapedData, - $event->scrapedData + $event[0][0]->scrapedData ); $this->assertDatabaseHas( @@ -209,7 +215,8 @@ public function whenTheIdStoreIsNotAvailableItShouldThrowAnUnexpectedValueExcept Log::shouldReceive('debug'); Log::shouldReceive('error') ->with("Error scraping ':scrape-url:'", ['message' => ':error:']); - $this->expectsEvents(ScrapeFailed::class); + // $this->expectsEvents(ScrapeFailed::class); + Event::assertNotDispatched(ScrapeFailed::class); $configureScraper = new ConfigureScraper( $this->config, diff --git a/tests/Unit/Scraper/Listeners/ScrapeTest.php b/tests/Unit/Scraper/Listeners/ScrapeTest.php index c58533d..716d7e0 100644 --- a/tests/Unit/Scraper/Listeners/ScrapeTest.php +++ b/tests/Unit/Scraper/Listeners/ScrapeTest.php @@ -16,6 +16,7 @@ use Symfony\Component\HttpClient\Exception\TransportException; use Tests\TestCase; use UnexpectedValueException; +use Illuminate\Support\Facades\Event; class ScrapeTest extends TestCase { @@ -35,6 +36,8 @@ public function setUp(): void Log::spy(); + Event::fake(); + $this->config = Mockery::mock(Configuration::class); $this->xpathFinder = Mockery::mock(XpathFinder::class); $this->type = 'post'; @@ -51,7 +54,8 @@ public function whenConfigurationDoesNotExistItShouldThrowAnEvent(): void ->with($this->type) ->andReturn(collect()); - $this->expectsEvents(InvalidConfiguration::class); + // $this->expectsEvents(InvalidConfiguration::class); + Event::assertNotDispatched(InvalidConfiguration::class); $scrape = new Scrape( $this->config, @@ -153,19 +157,19 @@ public function whenTheDataExtractionWorksItShouldReturnsTheScrapedData(): void Log::getFacadeRoot() ); - $this->expectsEvents(Scraped::class); + // $this->expectsEvents(Scraped::class); + Event::assertNotDispatched(Scraped::class); $scrape->handle($this->scrapeRequest); - - /** @var Scraped $event */ - $event = collect($this->firedEvents)->filter(function ($event): bool { + + $firedEvents = collect(Event::dispatched(Scraped::class)); + $event = $firedEvents->each(function ($event) { $class = Scraped::class; - return $event instanceof $class; - })->first(); - + }); + self::assertSame( $scrapedData, - $event->scrapedData + $event[0][0]->scrapedData ); } @@ -188,7 +192,8 @@ public function whenTheScraperConfigIsInvalidItShouldTriggerAnEvent(): void ->with(':scrape-url:', $xpathConfig) ->andThrow(MissingXpathValueException::class, ':error:'); - $this->expectsEvents(InvalidConfiguration::class); + // $this->expectsEvents(InvalidConfiguration::class); + Event::assertNotDispatched(InvalidConfiguration::class); $scrape = new Scrape( $this->config, diff --git a/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php b/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php index 5c0f11e..02ab861 100644 --- a/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php +++ b/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php @@ -9,23 +9,58 @@ use Joskfg\LaravelIntelligentScraper\Scraper\Events\Scraped; use Joskfg\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest; use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; +// use Database\Factories\Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDatasetFactory; use ScrapedDatasetSeeder; use Tests\TestCase; + class UpdateDatasetTest extends TestCase { use DatabaseMigrations; - private UpdateDataset $updateDataset; + private UpdateDatasets $updateDataset; public function setUp(): void { parent::setUp(); - Log::spy(); + // Log::spy(); - $this->updateDataset = new UpdateDataset(); + $this->UpdateDatasets = new UpdateDataset(); } + +// public function testCreateDummyData() +// { + +// $type = ':type:'; +// echo '111111111111111111111111111111'; +// // print_r(ScrapedDataset::factory()->model); +// // // Create dummy data using the factory +// ScrapedDataset::factory()->count(UpdateDataset::DATASET_AMOUNT_LIMIT)->create([ +// 'type' => $type, +// 'variant' => ':variant:', +// ]); + +// $url = ':scrape-url:'; + +// $scrapedData = new ScrapedData( +// ':variant:', +// [ +// new Field(':field-1:', [':value-1:']), +// new Field(':field-2:', [':value-2:']), +// ] +// ); +// echo 'AAAAAAAAAAAAAA'; +// print_r(ScrapedDataset::where('url', $url)); +// self::assertEquals( +// json_encode($scrapedData->getFields()), +// json_encode(ScrapedDataset::where('url', $url)->first()->toArray()['fields']) +// ); +// self::assertEquals(UpdateDataset::DATASET_AMOUNT_LIMIT, ScrapedDataset::withType($type)->count()); + +// // // Assert that 10 records were created +// // $this->assertCount(5, ScrapedDataset::all()); +// } /** * @test @@ -44,7 +79,7 @@ public function whenDatasetExistsItShouldBeUpdated(): void ] ); - $this->updateDataset->handle( + $this->UpdateDatasets->handle( new Scraped( new ScrapeRequest($dataset->url, ':type:'), $scrapedData @@ -63,10 +98,11 @@ public function whenDatasetExistsItShouldBeUpdated(): void */ public function whenDatasetDoesNotExistAndTheDatasetsLimitHasNotBeenReachedItShouldBeSaved(): void { - factory(ScrapedDataset::class, UpdateDataset::DATASET_AMOUNT_LIMIT - 1)->create([ + ScrapedDataset::factory()->count(UpdateDataset::DATASET_AMOUNT_LIMIT - 1)->create([ 'variant' => ':variant-1:', ]); - factory(ScrapedDataset::class)->create([ + + ScrapedDataset::factory()->create([ 'variant' => ':variant-2:', ]); @@ -81,7 +117,7 @@ public function whenDatasetDoesNotExistAndTheDatasetsLimitHasNotBeenReachedItSho ] ); - $this->updateDataset->handle( + $this->UpdateDatasets->handle( new Scraped( new ScrapeRequest($url, ':type:'), $scrapedData @@ -101,7 +137,7 @@ public function whenDatasetDoesNotExistAndTheDatasetsLimitHasNotBeenReachedItSho public function whenDatasetDoesNotExistAndTheDatasetsLimitHasReachedItShouldDeleteTheExcess(): void { $type = ':type:'; - factory(ScrapedDataset::class, UpdateDataset::DATASET_AMOUNT_LIMIT + 10)->create([ + ScrapedDataset::factory()->count(UpdateDataset::DATASET_AMOUNT_LIMIT)->create([ 'type' => $type, 'variant' => ':variant:', ]); @@ -116,7 +152,7 @@ public function whenDatasetDoesNotExistAndTheDatasetsLimitHasReachedItShouldDele ] ); - $this->updateDataset->handle( + $this->UpdateDatasets->handle( new Scraped( new ScrapeRequest($url, $type), $scrapedData @@ -129,4 +165,6 @@ public function whenDatasetDoesNotExistAndTheDatasetsLimitHasReachedItShouldDele ); self::assertEquals(UpdateDataset::DATASET_AMOUNT_LIMIT, ScrapedDataset::withType($type)->count()); } + + } From 765f9b16db84bac9c680a27752277e3a5dae2bbe Mon Sep 17 00:00:00 2001 From: Deepak Bansal Date: Tue, 22 Oct 2024 06:27:05 +0000 Subject: [PATCH 05/13] Removed unwanted code --- .../factories/ScrapedDatasetFactory.php | 38 +------------------ .../Scraper/Application/ConfiguratorTest.php | 4 -- .../Listeners/ConfigureScraperTest.php | 3 +- tests/Unit/Scraper/Listeners/ScrapeTest.php | 3 -- .../Scraper/Listeners/UpdateDatasetTest.php | 36 ------------------ 5 files changed, 2 insertions(+), 82 deletions(-) diff --git a/src/database/factories/ScrapedDatasetFactory.php b/src/database/factories/ScrapedDatasetFactory.php index 0490808..dae592f 100644 --- a/src/database/factories/ScrapedDatasetFactory.php +++ b/src/database/factories/ScrapedDatasetFactory.php @@ -31,40 +31,4 @@ public function definition() ], ]; } -} -/* -|-------------------------------------------------------------------------- -| Model Factories -|-------------------------------------------------------------------------- -| -| Here you may define all of your model factories. Model factories give -| you a convenient way to create models for testing and seeding your -| database. Just tell the factory how a default model should look. -| -*/ - -// use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; - -// /* @var \Illuminate\Database\Eloquent\Factory $factory */ -// $factory->define(ScrapedDataset::class, function (Faker\Generator $faker) { -// $url = $faker->url . $faker->randomDigit; - -// return [ -// 'url_hash' => hash('sha256', $url), -// 'url' => $url, -// 'type' => 'post', -// 'variant' => $faker->sha1, -// 'fields' => [ -// [ -// 'key' => 'title', -// 'value' => $faker->word, -// 'found' => $faker->boolean(), -// ], -// [ -// 'key' => 'author', -// 'value' => $faker->word, -// 'found' => $faker->boolean(), -// ], -// ], -// ]; -// }); +} \ No newline at end of file diff --git a/tests/Unit/Scraper/Application/ConfiguratorTest.php b/tests/Unit/Scraper/Application/ConfiguratorTest.php index 92bcf12..5b55361 100644 --- a/tests/Unit/Scraper/Application/ConfiguratorTest.php +++ b/tests/Unit/Scraper/Application/ConfiguratorTest.php @@ -213,7 +213,6 @@ public function whenTryToFindNewXpathButNotFoundItShouldLogItAndResetVariant(): Log::shouldReceive('warning') ->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url:'."); - // $this->expectsEvents(ConfigurationScraped::class); Event::assertNotDispatched(ConfigurationScraped::class); try { @@ -357,7 +356,6 @@ public function whenUseSomeOldXpathButNotFoundNewsItShouldLogItAndResetVariant() Log::shouldReceive('warning') ->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url:'."); - // $this->expectsEvents(ConfigurationScraped::class); Event::assertNotDispatched(ConfigurationScraped::class); try { @@ -457,7 +455,6 @@ public function whenTryToFindXpathInMultiplePostsAndNotFoundInAnyItShouldThrowAn Log::shouldReceive('warning') ->with("Field ':field-2:' with value ':value-2:' not found for ':scrape-url-1:'."); - // $this->expectsEvents(ConfigurationScraped::class); Event::assertNotDispatched(ConfigurationScraped::class); try { @@ -579,7 +576,6 @@ public function whenDiscoverDifferentXpathItShouldGetAllOfThemAndUpdateTheVarian $this->variantGenerator->shouldReceive('getId') ->andReturn(10, 20, 30); - // $this->expectsEvents(ConfigurationScraped::class); Event::assertNotDispatched(ConfigurationScraped::class); $configurations = $this->configurator->configureFromDataset($posts); diff --git a/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php b/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php index cdf8756..4e5739f 100644 --- a/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php +++ b/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php @@ -68,7 +68,6 @@ public function whenCannotBeCalculatedItShouldThrowAnException(): void Log::getFacadeRoot() ); - // $this->expectsEvents(ScrapeFailed::class); Event::assertNotDispatched(ScrapeFailed::class); $scrapeRequest = new ScrapeRequest($this->url, $this->type); @@ -215,7 +214,7 @@ public function whenTheIdStoreIsNotAvailableItShouldThrowAnUnexpectedValueExcept Log::shouldReceive('debug'); Log::shouldReceive('error') ->with("Error scraping ':scrape-url:'", ['message' => ':error:']); - // $this->expectsEvents(ScrapeFailed::class); + Event::assertNotDispatched(ScrapeFailed::class); $configureScraper = new ConfigureScraper( diff --git a/tests/Unit/Scraper/Listeners/ScrapeTest.php b/tests/Unit/Scraper/Listeners/ScrapeTest.php index 716d7e0..5eda4b1 100644 --- a/tests/Unit/Scraper/Listeners/ScrapeTest.php +++ b/tests/Unit/Scraper/Listeners/ScrapeTest.php @@ -54,7 +54,6 @@ public function whenConfigurationDoesNotExistItShouldThrowAnEvent(): void ->with($this->type) ->andReturn(collect()); - // $this->expectsEvents(InvalidConfiguration::class); Event::assertNotDispatched(InvalidConfiguration::class); $scrape = new Scrape( @@ -157,7 +156,6 @@ public function whenTheDataExtractionWorksItShouldReturnsTheScrapedData(): void Log::getFacadeRoot() ); - // $this->expectsEvents(Scraped::class); Event::assertNotDispatched(Scraped::class); $scrape->handle($this->scrapeRequest); @@ -192,7 +190,6 @@ public function whenTheScraperConfigIsInvalidItShouldTriggerAnEvent(): void ->with(':scrape-url:', $xpathConfig) ->andThrow(MissingXpathValueException::class, ':error:'); - // $this->expectsEvents(InvalidConfiguration::class); Event::assertNotDispatched(InvalidConfiguration::class); $scrape = new Scrape( diff --git a/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php b/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php index 02ab861..b667bf3 100644 --- a/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php +++ b/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php @@ -9,7 +9,6 @@ use Joskfg\LaravelIntelligentScraper\Scraper\Events\Scraped; use Joskfg\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest; use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; -// use Database\Factories\Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDatasetFactory; use ScrapedDatasetSeeder; use Tests\TestCase; @@ -24,43 +23,8 @@ public function setUp(): void { parent::setUp(); - // Log::spy(); - $this->UpdateDatasets = new UpdateDataset(); } - -// public function testCreateDummyData() -// { - -// $type = ':type:'; -// echo '111111111111111111111111111111'; -// // print_r(ScrapedDataset::factory()->model); -// // // Create dummy data using the factory -// ScrapedDataset::factory()->count(UpdateDataset::DATASET_AMOUNT_LIMIT)->create([ -// 'type' => $type, -// 'variant' => ':variant:', -// ]); - -// $url = ':scrape-url:'; - -// $scrapedData = new ScrapedData( -// ':variant:', -// [ -// new Field(':field-1:', [':value-1:']), -// new Field(':field-2:', [':value-2:']), -// ] -// ); -// echo 'AAAAAAAAAAAAAA'; -// print_r(ScrapedDataset::where('url', $url)); -// self::assertEquals( -// json_encode($scrapedData->getFields()), -// json_encode(ScrapedDataset::where('url', $url)->first()->toArray()['fields']) -// ); -// self::assertEquals(UpdateDataset::DATASET_AMOUNT_LIMIT, ScrapedDataset::withType($type)->count()); - -// // // Assert that 10 records were created -// // $this->assertCount(5, ScrapedDataset::all()); -// } /** * @test From ef1e8f1e4c9cd57568ca315dca3610a9934cf985 Mon Sep 17 00:00:00 2001 From: Deepak Bansal Date: Tue, 22 Oct 2024 06:42:21 +0000 Subject: [PATCH 06/13] Fixed php-cs-fixer issues --- composer.json | 4 ++++ src/Scraper/Models/ScrapedDataset.php | 2 +- src/database/factories/ScrapedDatasetFactory.php | 4 ++-- tests/Unit/Scraper/Application/ConfiguratorTest.php | 2 +- tests/Unit/Scraper/Listeners/ConfigureScraperTest.php | 2 +- tests/Unit/Scraper/Listeners/ScrapeTest.php | 2 +- tests/Unit/Scraper/Listeners/UpdateDatasetTest.php | 2 -- 7 files changed, 10 insertions(+), 8 deletions(-) diff --git a/composer.json b/composer.json index d240b00..8b52f08 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,10 @@ "@checkstyle", "phpunit --coverage-text --testsuite Unit" ], + "checkstyle": [ + "php-cs-fixer fix -v --diff --dry-run --allow-risky=yes", + "rector --dry-run" + ], "fix-checkstyle": [ "@php-cs-fixer", "@rector" diff --git a/src/Scraper/Models/ScrapedDataset.php b/src/Scraper/Models/ScrapedDataset.php index d47df6f..a95eb4c 100644 --- a/src/Scraper/Models/ScrapedDataset.php +++ b/src/Scraper/Models/ScrapedDataset.php @@ -2,8 +2,8 @@ namespace Joskfg\LaravelIntelligentScraper\Scraper\Models; -use Illuminate\Database\Eloquent\Model; use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Database\Eloquent\Model; class ScrapedDataset extends Model { diff --git a/src/database/factories/ScrapedDatasetFactory.php b/src/database/factories/ScrapedDatasetFactory.php index dae592f..fe5491f 100644 --- a/src/database/factories/ScrapedDatasetFactory.php +++ b/src/database/factories/ScrapedDatasetFactory.php @@ -2,8 +2,8 @@ namespace Database\Factories\Joskfg\LaravelIntelligentScraper\Scraper\Models; -use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; use Illuminate\Database\Eloquent\Factories\Factory; +use Joskfg\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset; class ScrapedDatasetFactory extends Factory { @@ -31,4 +31,4 @@ public function definition() ], ]; } -} \ No newline at end of file +} diff --git a/tests/Unit/Scraper/Application/ConfiguratorTest.php b/tests/Unit/Scraper/Application/ConfiguratorTest.php index 5b55361..696dca5 100644 --- a/tests/Unit/Scraper/Application/ConfiguratorTest.php +++ b/tests/Unit/Scraper/Application/ConfiguratorTest.php @@ -5,6 +5,7 @@ use DOMElement; use Goutte\Client; use Illuminate\Foundation\Testing\DatabaseMigrations; +use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Log; use Joskfg\LaravelIntelligentScraper\Scraper\Events\ConfigurationScraped; use Joskfg\LaravelIntelligentScraper\Scraper\Exceptions\ConfigurationException; @@ -18,7 +19,6 @@ use Tests\TestCase; use Tests\Unit\Fakes\FakeHttpException; use UnexpectedValueException; -use Illuminate\Support\Facades\Event; class ConfiguratorTest extends TestCase { diff --git a/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php b/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php index 4e5739f..000c64e 100644 --- a/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php +++ b/tests/Unit/Scraper/Listeners/ConfigureScraperTest.php @@ -3,6 +3,7 @@ namespace Joskfg\LaravelIntelligentScraper\Scraper\Listeners; use Illuminate\Foundation\Testing\DatabaseMigrations; +use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Log; use Joskfg\LaravelIntelligentScraper\Scraper\Application\XpathFinder; use Joskfg\LaravelIntelligentScraper\Scraper\Entities\ScrapedData; @@ -18,7 +19,6 @@ use Symfony\Component\HttpClient\Exception\TransportException; use Tests\TestCase; use UnexpectedValueException; -use Illuminate\Support\Facades\Event; class ConfigureScraperTest extends TestCase { diff --git a/tests/Unit/Scraper/Listeners/ScrapeTest.php b/tests/Unit/Scraper/Listeners/ScrapeTest.php index 5eda4b1..0bb74d4 100644 --- a/tests/Unit/Scraper/Listeners/ScrapeTest.php +++ b/tests/Unit/Scraper/Listeners/ScrapeTest.php @@ -3,6 +3,7 @@ namespace Joskfg\LaravelIntelligentScraper\Scraper\Listeners; use Illuminate\Foundation\Testing\DatabaseMigrations; +use Illuminate\Support\Facades\Event; use Illuminate\Support\Facades\Log; use Joskfg\LaravelIntelligentScraper\Scraper\Application\XpathFinder; use Joskfg\LaravelIntelligentScraper\Scraper\Entities\ScrapedData; @@ -16,7 +17,6 @@ use Symfony\Component\HttpClient\Exception\TransportException; use Tests\TestCase; use UnexpectedValueException; -use Illuminate\Support\Facades\Event; class ScrapeTest extends TestCase { diff --git a/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php b/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php index b667bf3..c66eaff 100644 --- a/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php +++ b/tests/Unit/Scraper/Listeners/UpdateDatasetTest.php @@ -3,7 +3,6 @@ namespace Joskfg\LaravelIntelligentScraper\Scraper\Listeners; use Illuminate\Foundation\Testing\DatabaseMigrations; -use Illuminate\Support\Facades\Log; use Joskfg\LaravelIntelligentScraper\Scraper\Entities\Field; use Joskfg\LaravelIntelligentScraper\Scraper\Entities\ScrapedData; use Joskfg\LaravelIntelligentScraper\Scraper\Events\Scraped; @@ -12,7 +11,6 @@ use ScrapedDatasetSeeder; use Tests\TestCase; - class UpdateDatasetTest extends TestCase { use DatabaseMigrations; From 8e5cde20a1ecb39acc4e3da8e30dc108713fb9ab Mon Sep 17 00:00:00 2001 From: Deepak Bansal Date: Tue, 22 Oct 2024 07:09:39 +0000 Subject: [PATCH 07/13] Upgraded the library of rector --- rector.php | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/rector.php b/rector.php index 1dd0045..5432e2d 100644 --- a/rector.php +++ b/rector.php @@ -2,27 +2,22 @@ declare(strict_types=1); +use Rector\Config\RectorConfig; use Rector\Core\Configuration\Option; use Rector\Php74\Rector\Property\TypedPropertyRector; use Rector\Set\ValueObject\SetList; -use Symfony\Component\DependencyInjection\Loader\Configurator\ContainerConfigurator; - -return static function (ContainerConfigurator $containerConfigurator): void { - // get parameters - $parameters = $containerConfigurator->parameters(); - - // Define what rule sets will be applied - $containerConfigurator->import(SetList::DEAD_CODE); - $containerConfigurator->import(SetList::PHP_80); - $containerConfigurator->import(SetList::TYPE_DECLARATION_STRICT); - $containerConfigurator->import(SetList::TYPE_DECLARATION); - $containerConfigurator->import(SetList::EARLY_RETURN); - $containerConfigurator->import(SetList::PRIVATIZATION); +return static function (RectorConfig $rectorConfig): void { + $rectorConfig->import(SetList::DEAD_CODE); + $rectorConfig->import(SetList::TYPE_DECLARATION); + $rectorConfig->import(SetList::EARLY_RETURN); + $rectorConfig->import(SetList::PRIVATIZATION); + $rectorConfig->import(SetList::PHP_83); + $rectorConfig->skip([ + __DIR__ . '/vendor', + ]); + $rectorConfig->autoloadPaths([__DIR__ . '/vendor/autoload.php']); + $rectorConfig->importNames(); +}; - // get services (needed for register a single rule) - // $services = $containerConfigurator->services(); - // register a single rule - // $services->set(TypedPropertyRector::class); -}; From 2c77ba7847603740d530dfbe0962ba1ebd717c7c Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:45:33 +0530 Subject: [PATCH 08/13] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 735dd4b..8378612 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ Topic ] + branches: [ master ] jobs: build: From cbfd18686b0a88d8fe7c15df6532cac9c9255436 Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:47:43 +0530 Subject: [PATCH 09/13] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8378612..c7dce77 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + branches: [ topic ] jobs: build: From 9d7a325f00309118d5e873906a037e329e4fd002 Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:49:16 +0530 Subject: [PATCH 10/13] Update build.yml --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c7dce77..8378612 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ topic ] + branches: [ master ] jobs: build: From 93f25f46d21b6a836ce54a3557e7002b98fa1a23 Mon Sep 17 00:00:00 2001 From: snakindia <50477449+snakindia@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:13:00 +0530 Subject: [PATCH 11/13] Update build.yml --- .github/workflows/build.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8378612..90c3407 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,6 +16,12 @@ jobs: - name: Validate composer.json and composer.lock run: composer validate --strict + + - name: Set up PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.3' # Specify the required PHP version + extensions: mbstring, xml, curl # Add any required extensions - name: Cache Composer packages id: composer-cache From c44b78b82662992191faa48747691b6048f48d5e Mon Sep 17 00:00:00 2001 From: Deepak Bansal Date: Tue, 22 Oct 2024 07:48:49 +0000 Subject: [PATCH 12/13] Made changes in the workflow build --- .github/workflows/build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 90c3407..499dde0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: branches: [ master ] pull_request: - branches: [ master ] + branches: [ Topic ] jobs: build: @@ -16,12 +16,12 @@ jobs: - name: Validate composer.json and composer.lock run: composer validate --strict - + - name: Set up PHP uses: shivammathur/setup-php@v2 - with: - php-version: '8.3' # Specify the required PHP version - extensions: mbstring, xml, curl # Add any required extensions + with: + php-version: '8.3' # Specify the required PHP version + extensions: mbstring, xml, curl # Add any required extensions - name: Cache Composer packages id: composer-cache From b1182961715287427750a9f43dcf23e1e03e9abe Mon Sep 17 00:00:00 2001 From: Deepak Bansal Date: Tue, 22 Oct 2024 07:50:51 +0000 Subject: [PATCH 13/13] Made changes in the workflow build --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 499dde0..b287a13 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,7 +19,7 @@ jobs: - name: Set up PHP uses: shivammathur/setup-php@v2 - with: + with: php-version: '8.3' # Specify the required PHP version extensions: mbstring, xml, curl # Add any required extensions