MDTT-10: Support JSON for source and destination (#12)

README.md

@@ -62,13 +62,13 @@ description: Validates users
 group: migration_validation
 # The query used for generating the source dataset
 source:
-  type: query
+  type: database
   data: "select * from users"
   # The source database credentials
   database: source_db
 # The query used for generating the destination dataset
 destination:
-  type: query
+  type: database
   data: "select * from users_field_data"
   # The destination database credentials
   database: destination_db
@@ -84,6 +84,35 @@ tests:
     destinationField: mail
 ```
 
+#### JSON
+
+```yaml
+id: validate_public_apis
+description: Validate public apis
+group: migration_validation
+# The endpoint that returns the source dataset.
+source:
+  type: json
+  data: https://dev.legacy-system.com/api/v1/users
+  # The pointer where all the list of items resides. Refer https://github.com/halaxa/json-machine#what-is-json-pointer-anyway for examples
+  selector: "/results/-/employees"
+  # Basic authentication credentials to access the endpoint. This is optional if the endpoint is publicly accessible.
+  auth_basic: "foo:bar"
+# The endpoint that returns the destination dataset.
+destination:
+  type: json
+  data: https://dev.new-system.com/api/v1/users
+  selector: "/results/-/employees"
+  auth_basic: "foo:bar"
+tests:
+  -
+    sourceField: name
+    destinationField: name
+  -
+    sourceField: email
+    destinationField: email
+```
+
 You can find the basic template for the tool usage [here](https://github.com/axelerant/mdtt-usage).
 
 ## Run tests
@@ -109,10 +138,12 @@ You can find the basic template for the tool usage [here](https://github.com/axe
 ## Supported types of source
 
 - Database (MySQL)
+- JSON
 
 ## Supported types of destination
 
 - Database (MySQL)
+- JSON
 
 ## Supported types of channels to notify when test completes
 

composer.json

@@ -20,7 +20,9 @@
         "phpunit/phpunit": "^9.5",
         "symfony/dotenv": "^6.0",
         "sendgrid/sendgrid": "^7.11",
-        "monolog/monolog": "^2.4"
+        "monolog/monolog": "^2.4",
+        "guzzlehttp/guzzle": "^7.4",
+        "halaxa/json-machine": "^1.1"
     },
     "scripts": {
         "php-cs": "./vendor/bin/phpcs src --standard=PSR2",

src/DataSource.php

@@ -2,6 +2,8 @@
 
 namespace Mdtt;
 
+use Iterator;
+
 abstract class DataSource
 {
     protected string $data;
@@ -14,7 +16,7 @@ public function __construct(string $data)
     /**
      * Returns an item from the source.
      *
-     * @return array<string>|array<int>
+     * @return Iterator
      */
-    abstract public function getItem(): ?array;
+    abstract public function getItem(): Iterator;
 }

src/Definition/DefaultDefinition.php

@@ -132,25 +132,25 @@ public function runTests(): void
     {
         $source = $this->getSource();
         $destination = $this->getDestination();
-        $this->logger->info("Running the tests now...");
+        $this->logger->info(sprintf("Running the tests of definition id: %s", $this->id));
 
-        /** @var array<scalar>|null $sourceData */
         $sourceData = $source->getItem();
-        /** @var array<scalar>|null $destinationData */
         $destinationData = $destination->getItem();
 
-        while ($sourceData && $destinationData) {
+        // Combining the iterators is required so that the tests can be run for every returned item.
+        $combinedDataSources = new \MultipleIterator();
+        $combinedDataSources->attachIterator($sourceData);
+        $combinedDataSources->attachIterator($destinationData);
+
+        foreach ($combinedDataSources as [$sourceValue, $destinationValue]) {
             foreach ($this->getTests() as $test) {
-                $test->execute($sourceData, $destinationData);
+                $test->execute($sourceValue, $destinationValue);
             }
-
-            $sourceData = $source->getItem();
-            $destinationData = $destination->getItem();
         }
 
         try {
             Assert::assertTrue(
-                ($sourceData === null) && ($destinationData === null),
+                !$sourceData->valid() && !$destinationData->valid(),
                 "Number of source items does not match number of destination items."
             );
         } catch (ExpectationFailedException $exception) {

src/Definition/Validate/DataSource/Database.php

@@ -0,0 +1,16 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mdtt\Definition\Validate\DataSource;
+
+class Database implements Type
+{
+    /**
+     * @inheritDoc
+     */
+    public function validate(array $rawDataSourceDefinition): bool
+    {
+        return isset($rawDataSourceDefinition['database']);
+    }
+}

src/Definition/Validate/DataSource/Json.php

@@ -0,0 +1,16 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mdtt\Definition\Validate\DataSource;
+
+class Json implements Type
+{
+    /**
+     * @inheritDoc
+     */
+    public function validate(array $rawDataSourceDefinition): bool
+    {
+        return isset($rawDataSourceDefinition['selector']);
+    }
+}

src/Definition/Validate/DataSource/Type.php

@@ -0,0 +1,15 @@
+<?php
+
+namespace Mdtt\Definition\Validate\DataSource;
+
+interface Type
+{
+    /**
+     * Validates whether all required information are mentioned in the datasource definition.
+     *
+     * @param array<string> $rawDataSourceDefinition
+     *
+     * @return bool
+     */
+    public function validate(array $rawDataSourceDefinition): bool;
+}

src/Definition/Validate/DataSource/Validator.php

@@ -0,0 +1,116 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mdtt\Definition\Validate\DataSource;
+
+use Mdtt\DataSource;
+use Mdtt\Exception\SetupException;
+use Mdtt\Utility\DataSource\Json as JsonDataSourceUtility;
+
+class Validator
+{
+    private JsonDataSourceUtility $jsonDataSourceUtility;
+
+    public function __construct(JsonDataSourceUtility $jsonDataSourceUtility)
+    {
+        $this->jsonDataSourceUtility = $jsonDataSourceUtility;
+    }
+
+    /**
+     * Validates whether the datasource information in test definition is valid.
+     *
+     * @param string $type
+     * @param array<string> $rawDataSourceDefinition
+     *
+     * @return \Mdtt\DataSource
+     * @throws \Mdtt\Exception\SetupException
+     */
+    public function validate(string $type, array $rawDataSourceDefinition): DataSource
+    {
+        if (!in_array($type, ["source", "destination"])) {
+            throw new SetupException("Incorrect data source type is passed.");
+        }
+
+        /** @var string $dataSourceType */
+        $dataSourceType = $rawDataSourceDefinition['type'];
+        if ($dataSourceType === "database") {
+            $this->doValidateDatabase($rawDataSourceDefinition);
+
+            if ($type === "source") {
+                return new \Mdtt\Source\Database(
+                    $rawDataSourceDefinition['data'],
+                    $rawDataSourceDefinition['database']
+                );
+            }
+
+            if ($type === "destination") {
+                return new \Mdtt\Destination\Database(
+                    $rawDataSourceDefinition['data'],
+                    $rawDataSourceDefinition['database']
+                );
+            }
+        }
+
+        if ($dataSourceType === "json") {
+            $this->doValidateJson($rawDataSourceDefinition);
+
+            if (isset($rawDataSourceDefinition['auth_basic'])) {
+                $this->jsonDataSourceUtility->setAuthBasicCredential($rawDataSourceDefinition['auth_basic']);
+            }
+
+            if ($type === "source") {
+                return new \Mdtt\Source\Json(
+                    $rawDataSourceDefinition['data'],
+                    $rawDataSourceDefinition['selector'],
+                    $this->jsonDataSourceUtility,
+                );
+            }
+
+            if ($type === "destination") {
+                return new \Mdtt\Destination\Json(
+                    $rawDataSourceDefinition['data'],
+                    $rawDataSourceDefinition['selector'],
+                    $this->jsonDataSourceUtility
+                );
+            }
+        }
+
+        throw new SetupException(sprintf("Unexpected data source type %s and data source definition passed.", $type));
+    }
+
+    /**
+     * @param array<string> $rawDataSourceDefinition
+     *
+     * @return void
+     */
+    private function doValidateDatabase(array $rawDataSourceDefinition): void
+    {
+        $dbValidator = new Database();
+        $isValid = $dbValidator->validate($rawDataSourceDefinition);
+        if (!$isValid) {
+            throw new SetupException(
+                sprintf("All information are not passed for %s", $rawDataSourceDefinition['type'])
+            );
+        }
+    }
+
+    /**
+     * @param array<string> $rawDataSourceDefinition
+     *
+     * @return void
+     */
+    private function doValidateJson(array $rawDataSourceDefinition): void
+    {
+        $jsonValidator = new Json();
+        $isValid = $jsonValidator->validate($rawDataSourceDefinition);
+        if (!$isValid) {
+            throw new SetupException(
+                sprintf(
+                    "All information are not passed for %s",
+                    $rawDataSourceDefinition['type']
+                )
+            );
+        }
+    }
+}

src/Destination/Database.php

@@ -4,8 +4,8 @@
 
 namespace Mdtt\Destination;
 
+use Iterator;
 use Mdtt\DataSource;
-use Mdtt\Exception\ExecutionException;
 use Mdtt\Exception\SetupException;
 use Mdtt\Utility\DataSource\Database as DbDatabase;
 use mysqli_result;
@@ -24,7 +24,7 @@ public function __construct(string $data, string $databaseKey)
     /**
      * @inheritDoc
      */
-    public function getItem(): ?array
+    public function getItem(): Iterator
     {
         $specification = require "tests/mdtt/spec.php";
 
@@ -51,13 +51,8 @@ public function getItem(): ?array
             );
         }
 
-        /** @var array<int|string>|false|null $row */
-        $row = mysqli_fetch_assoc($this->resultSet);
-
-        if ($row === false) {
-            throw new ExecutionException("Something went wrong while retrieving an item from the destination.");
+        while ($row = $this->resultSet->fetch_assoc()) {
+            yield $row;
         }
-
-        return $row;
     }
 }

src/Destination/Json.php

@@ -0,0 +1,49 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Mdtt\Destination;
+
+use Iterator;
+use JsonMachine\Items;
+use Mdtt\DataSource;
+use Mdtt\Utility\DataSource\Json as JsonDataSourceUtility;
+
+class Json extends DataSource
+{
+    private string $selector;
+    private JsonDataSourceUtility $jsonDataSourceUtility;
+    private Items|null $items;
+
+    public function __construct(
+        string $data,
+        string $selector,
+        JsonDataSourceUtility $jsonDataSourceUtility
+    ) {
+        parent::__construct($data);
+        $this->selector = $selector;
+        $this->jsonDataSourceUtility = $jsonDataSourceUtility;
+    }
+
+    /**
+     * @return string
+     */
+    public function getSelector(): string
+    {
+        return $this->selector;
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function getItem(): Iterator
+    {
+        if (!isset($this->items)) {
+            $this->items = $this->jsonDataSourceUtility->getItems($this->data, $this->selector);
+        }
+
+        foreach ($this->items as $item) {
+            yield $item;
+        }
+    }
+}