diff --git a/app/Http/Controllers/Admin/Workflow/BulkPublishingController.php b/app/Http/Controllers/Admin/Workflow/BulkPublishingController.php index e232881ffd..acfa635835 100644 --- a/app/Http/Controllers/Admin/Workflow/BulkPublishingController.php +++ b/app/Http/Controllers/Admin/Workflow/BulkPublishingController.php @@ -109,6 +109,7 @@ public function checkCoreElementsCompleted(Request $request): JsonResponse return response()->json(['success' => false, 'message' => trans('responses.no_activities_selected')]); } catch (\Exception $e) { logger()->error($e->getMessage()); + logger()->error($e); return response()->json(['success' => false, 'message' => trans('responses.error_has_occurred', ['event'=>trans('events.checking'), 'suffix'=>trans('common.core_completed_title')])]); } @@ -153,6 +154,7 @@ public function validateActivities(Request $request): JsonResponse } catch (\Exception $e) { DB::rollBack(); logger()->error($e->getMessage()); + logger()->error($e); return response()->json(['success' => false, 'message' => trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('elements_common.activities')])]); } @@ -169,10 +171,12 @@ public function validateActivities(Request $request): JsonResponse */ public function startBulkPublish(Request $request): JsonResponse { + logger()->error('starting bulk publish entered'); try { DB::beginTransaction(); + $organization = auth()->user()->organization; - $message = $this->activityWorkflowService->getPublishErrorMessage(auth()->user()->organization); + $message = $this->activityWorkflowService->getPublishErrorMessage($organization); if (!empty($message)) { Session::put('error', $message); @@ -180,7 +184,9 @@ public function startBulkPublish(Request $request): JsonResponse return response()->json(['success' => false, 'message' => $message]); } - if ($this->publishingStatusService->ongoingBulkPublishing(auth()->user()->organization->id)) { + logger()->error('checking bulk publishing is ongoing'); + + if ($this->publishingStatusService->ongoingBulkPublishing($organization->id)) { $pubishingStatus = $this->bulkPublishingService->getOrganisationBulkPublishingStatus(); return response()->json([ @@ -194,15 +200,18 @@ public function startBulkPublish(Request $request): JsonResponse $activityIds = json_decode($request->get('activities'), false, 512, JSON_THROW_ON_ERROR); if (!empty($activityIds)) { + logger()->error('fetching activities'); $activities = $this->activityService->getActivitiesHavingIds($activityIds); if (!count($activities)) { return response()->json(['success' => false, 'message' => trans('responses.no_activities_selected')]); } - + logger()->error('storing initial response '); $response = $this->bulkPublishingService->generateInitialBulkPublishingResponse($activities); $this->publishingStatusService->storeProcessingActivities($activities, $response['organization_id'], $response['job_batch_uuid']); - dispatch(new BulkPublishActivities($activities, $response['organization_id'], $response['job_batch_uuid'])); + logger()->error('start bulk publishing'); + dispatch(new BulkPublishActivities($activities, $organization, $organization->settings, $response['organization_id'], $response['job_batch_uuid'])); + logger()->error('bulk publishing completed'); DB::commit(); return response()->json(['success' => true, 'message' => trans('responses.bulk_publishing', ['suffix'=>trans('responses.started')]), 'data' => $response]); @@ -212,6 +221,7 @@ public function startBulkPublish(Request $request): JsonResponse } catch (\Exception $e) { DB::rollBack(); logger()->error($e->getMessage()); + logger()->error($e); return response()->json(['success' => false, 'message' => trans('responses.bulk_publishing', ['suffix'=>trans('responses.failed')])]); } @@ -280,6 +290,7 @@ public function cancelBulkPublishing(): JsonResponse } catch (\Exception $e) { DB::rollBack(); logger()->error($e->getMessage()); + logger()->error($e); return response()->json(['success' => false, 'message' => trans('responses.no_bulk_publish_were_cancelled')]); } @@ -313,6 +324,7 @@ public function checksForActivityBulkPublish(): JsonResponse return response()->json(['success' => true, 'message' => trans('responses.activity_ready_to_publish'), 'status' => 'completed']); } catch (\Exception $e) { logger()->error($e->getMessage()); + logger()->error($e); return response()->json(['success' => false, 'message' => trans('responses.error_has_occurred', ['event'=>trans('events.checking'), 'suffix'=>trans('common.activity')])]); } @@ -334,6 +346,7 @@ public function clearBulkPublishStatus(): JsonResponse return response()->json(['success' => true, 'message' => 'Bulk publishing status successfully deleted.']); } catch (\Exception $e) { logger()->error($e->getMessage()); + logger()->error($e); DB::rollBack(); return response()->json(['success' => false, 'message' => 'Failed to delete bulk publishing status']); diff --git a/app/IATI/Elements/Xml/XmlGenerator.php b/app/IATI/Elements/Xml/XmlGenerator.php index 2145e1d707..1a43ab7ed1 100644 --- a/app/IATI/Elements/Xml/XmlGenerator.php +++ b/app/IATI/Elements/Xml/XmlGenerator.php @@ -241,7 +241,7 @@ public function __construct( } /** - * Generates combines activities xml file and publishes to IATI. + * Generates single activity and combines xml file and publishes to IATI. * * @param $activity * @param $transaction @@ -265,7 +265,52 @@ public function generateActivityXml($activity, $transaction, $result, $settings, if ($result) { $publishedFiles = $this->savePublishedFiles($filename, $activity->org_id, $publishedActivity); + logger()->error('saving published file'); $this->getMergeXml($publishedFiles, $filename); + logger()->error('merged activity'); + } + } + + /** + * Generates multiple activities and combines xml file and publishes to IATI. + * + * @param $activity + * @param $transaction + * @param $result + * @param $settings + * @param $organization + * + * @return void + */ + public function generateActivitiesXml($activityData, $settings, $organization) + { + $publishedFiles = []; + $publishingInfo = $settings->publishing_info; + $publisherId = Arr::get($publishingInfo, 'publisher_id', 'Not Available'); + $filename = sprintf('%s-%s.xml', $publisherId, 'activities'); + + foreach ($activityData as $activity) { + $publishedActivity = sprintf('%s-%s.xml', $publisherId, $activity->id); + $xml = $this->getXml($activity, $activity->transaction ?? [], $activity->results ?? [], $settings, $organization); + $result = awsUploadFile( + sprintf('%s/%s/%s', 'xml', 'activityXmlFiles', $publishedActivity), + $xml->saveXML() + ); + + $publishedFiles[] = $publishedActivity; + + // $publishedFiles = $this->savePublishedFiles($filename, $activity->org_id, $publishedActivity); + } + + if (count($publishedFiles)) { + // foreach ($publishedFiles as $file) { + $filesToMerge = $this->savePublishedFiles($filename, $activity->org_id, $publishedFiles); + // } + // dd($filesToMerge); + + logger()->error('saving published file'); + $this->getMergeXml($filesToMerge, $filename); + logger()->error('merged activity'); } } @@ -282,15 +327,21 @@ public function savePublishedFiles($filename, $organizationId, $publishedActivit { $published = $this->activityPublishedService->findOrCreate($filename, $organizationId); $publishedActivities = $publishedActivity; + // dd($publishedActivities, $published->published_activities); if (!is_array($publishedActivity)) { $publishedActivities = (array) $published->published_activities; (in_array($publishedActivity, $publishedActivities, true)) ?: array_push($publishedActivities, $publishedActivity); } + if (is_array($publishedActivity)) { + $publishedActivities = (array) $published->published_activities; + $publishedActivities = array_merge($publishedActivities, $publishedActivity); + } + $this->activityPublishedService->update($published, array_unique($publishedActivities)); - return $published->published_activities; + return $publishedActivities; } /** @@ -312,6 +363,7 @@ public function getMergeXml($publishedFiles, $filename): void $iatiActivities->appendChild($dom->createComment('Generated By IATI Publisher')); foreach ($publishedFiles as $xml) { + logger()->error('xml'); $addDom = new \DOMDocument(); $fileContent = awsGetFile(sprintf('%s/%s/%s', 'xml', 'activityXmlFiles', $xml)); @@ -392,12 +444,12 @@ public function getXmlAttributes($defaultValues): array { $data = [ 'last-updated-datetime' => gmdate('c', time()), - 'xml:lang' => Arr::get($defaultValues, 'default_language', null), - 'default-currency' => Arr::get($defaultValues, 'default_currency', null), - 'humanitarian' => Arr::get($defaultValues, 'humanitarian', 1), - 'hierarchy' => Arr::get($defaultValues, 'hierarchy', 1), - 'budget-not-provided' => Arr::get($defaultValues, 'budget_not_provided', ''), - 'linked-data-uri' => Arr::get($defaultValues, 'linked_data_uri', ''), + 'xml:lang' => Arr::get($defaultValues, 'default_language', null), + 'default-currency' => Arr::get($defaultValues, 'default_currency', null), + 'humanitarian' => Arr::get($defaultValues, 'humanitarian', 1), + 'hierarchy' => Arr::get($defaultValues, 'hierarchy', 1), + 'budget-not-provided' => Arr::get($defaultValues, 'budget_not_provided', ''), + 'linked-data-uri' => Arr::get($defaultValues, 'linked_data_uri', ''), ]; foreach ($data as $key => $datum) { @@ -463,6 +515,7 @@ public function setServices(): void */ public function getXmlData($activity, $transaction, $result, $organization): array { + logger()->error('starting xml data fetching'); $xmlActivity = []; $xmlActivity['iati-identifier'] = ($organization->identifier ?: 'Not Available') . '-' . Arr::get($activity->iati_identifier, 'activity_identifier', 'Not Available'); $xmlActivity['reporting-org'] = $this->reportingOrgService->getXmlData($activity); @@ -499,6 +552,7 @@ public function getXmlData($activity, $transaction, $result, $organization): arr removeEmptyValues($xmlActivity); $this->mapActivityTransactionAndResultIndex($xmlActivity, $activity); + logger()->error('xml data fetched'); return $xmlActivity; } diff --git a/app/IATI/Services/Publisher/PublisherService.php b/app/IATI/Services/Publisher/PublisherService.php index 5fc3cebe11..a676232fa7 100644 --- a/app/IATI/Services/Publisher/PublisherService.php +++ b/app/IATI/Services/Publisher/PublisherService.php @@ -52,11 +52,13 @@ public function __construct(ActivityPublishedService $activityPublishedService, */ public function publishFile($registryInfo, $activityPublished, $organization, bool $updatedActivityPublished = true): void { + logger()->error('publishing file stated'); $this->setFile($activityPublished); $this->init(env('IATI_API_ENDPOINT'), Arr::get($registryInfo, 'api_token', '')) ->setPublisher(Arr::get($registryInfo, 'publisher_id', '')); $this->searchForPublisher($this->publisherId); $this->publishToRegistry($organization, $activityPublished->filename, $updatedActivityPublished); + logger()->error('file published'); } /** diff --git a/app/IATI/Services/Workflow/ActivityWorkflowService.php b/app/IATI/Services/Workflow/ActivityWorkflowService.php index 2e81262369..5994061695 100644 --- a/app/IATI/Services/Workflow/ActivityWorkflowService.php +++ b/app/IATI/Services/Workflow/ActivityWorkflowService.php @@ -126,6 +126,7 @@ public function findActivity($activityId): object */ public function publishActivity($activity, bool $publishFile = true): void { + logger()->error('publishing activity'); $organization = $activity->organization; $settings = $organization->settings; $this->xmlGeneratorService->generateActivityXml( @@ -135,15 +136,53 @@ public function publishActivity($activity, bool $publishFile = true): void $settings, $organization ); + logger()->error('activity xml generation'); if ($publishFile) { + logger()->error('publising xml file'); $activityPublished = $this->activityPublishedService->getActivityPublished($organization->id); $publishingInfo = $settings->publishing_info; $this->publisherService->publishFile($publishingInfo, $activityPublished, $organization); + logger()->error('xml file published'); } $this->activityService->updatePublishedStatus($activity, 'published', true); + logger()->error('updated publisher status'); $this->activitySnapshotService->createOrUpdateActivitySnapshot($activity); + logger()->error('updated snapshot'); + } + + /** + * Publish an activities to the IATI registry. + * + * @param $activities + * @param $organization + * @param $settings + * @param bool $publishFile + * + * @return void + */ + public function publishActivities($activities, $organization, $settings, bool $publishFile = true): void + { + logger()->error('activity xml generation'); + $this->xmlGeneratorService->generateActivitiesXml( + $activities, + $settings, + $organization + ); + + logger()->error('publising xml file'); + $activityPublished = $this->activityPublishedService->getActivityPublished($organization->id); + $publishingInfo = $settings->publishing_info; + $this->publisherService->publishFile($publishingInfo, $activityPublished, $organization); + logger()->error('xml file published'); + + foreach ($activities as $activity) { + $this->activityService->updatePublishedStatus($activity, 'published', true); + logger()->error('updated publisher status'); + $this->activitySnapshotService->createOrUpdateActivitySnapshot($activity); + logger()->error('updated snapshot'); + } } /** @@ -202,6 +241,7 @@ public function validateActivityOnIATIValidator($activity): string $settings, $organization ); + logger()->error("xmlValidation/$activity->org_id/activity_$activity->id.xml created"); awsUploadFile("xmlValidation/$activity->org_id/activity_$activity->id.xml", $xmlData); @@ -220,6 +260,7 @@ public function validateActivityOnIATIValidator($activity): string */ public function getResponse($xmlData): string { + file_put_contents(storage_path('test.xml'), $xmlData); $client = new Client(); $URI = env('IATI_VALIDATOR_ENDPOINT'); $params['headers'] = ['Content-Type' => 'application/json', 'Ocp-Apim-Subscription-Key' => env('IATI_VALIDATOR_KEY')]; @@ -340,7 +381,8 @@ public function populateSectorIfMissing($activity): object { $data = sectorDefaultValue(); - if ((empty($activity->sector) && !$this->activityService->checkIfTransactionHasSector($activity)) + if ( + (empty($activity->sector) && !$this->activityService->checkIfTransactionHasSector($activity)) || (is_variable_null($activity->sector)) ) { $this->sectorService->update($activity->id, $data); diff --git a/app/IATI/Services/Workflow/BulkPublishingService.php b/app/IATI/Services/Workflow/BulkPublishingService.php index 7da910d386..7eb034a57c 100644 --- a/app/IATI/Services/Workflow/BulkPublishingService.php +++ b/app/IATI/Services/Workflow/BulkPublishingService.php @@ -133,6 +133,7 @@ public function validateActivitiesOnIATI($activityIds): array if ($activity && $activity->status === 'draft') { $response = $this->validateWithException($activity); $this->apiLogRepo->store(generateApiInfo('POST', env('IATI_VALIDATOR_ENDPOINT'), ['form_params' => json_encode($activity)], json_encode($response))); + logger()->error('activity validated'); if (!Arr::get($response, 'success', true)) { logger()->error(trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('responses.activity_with_id')]) . $activityId); @@ -160,28 +161,38 @@ public function validateActivitiesOnIATI($activityIds): array */ public function validateWithException($activity): array { + logger()->error('validating with exception'); try { $validatorResponse = $this->activityWorkflowService->validateActivityOnIATIValidator($activity); + logger()->error('validator response'); + // $response = $validatorResponse; $response = $this->addElementOnIatiValidatorResponse($validatorResponse, $activity); - + logger()->error('validator response attested'); if ($this->validatorService->updateOrCreateResponse($activity->id, $response)) { return $response; } + logger()->error('validator response saved'); return ['success' => false, 'error' => trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('elements_common.activity')])]; } catch (BadResponseException $ex) { + logger()->error('client error'); if ($ex->getCode() === 422) { $validatorResponse = $ex->getResponse()->getBody()->getContents(); - $response = $this->addElementOnIatiValidatorResponse($validatorResponse, $activity); + $response = $validatorResponse; + $response = json_decode($validatorResponse, true, 512, JSON_THROW_ON_ERROR); + + // $response = $this->addElementOnIatiValidatorResponse($validatorResponse, $activity); if ($this->validatorService->updateOrCreateResponse($activity->id, $response)) { return $response; } } + logger()->error('completed'); return ['success' => false, 'error' => trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('elements_common.activity')])]; } catch (\Exception $e) { logger()->error($e->getMessage()); + logger()->error($e); throw new \RuntimeException(); } diff --git a/app/IATI/Services/Xml/XmlGeneratorService.php b/app/IATI/Services/Xml/XmlGeneratorService.php index af7a055869..ce5c43b12f 100644 --- a/app/IATI/Services/Xml/XmlGeneratorService.php +++ b/app/IATI/Services/Xml/XmlGeneratorService.php @@ -52,6 +52,20 @@ public function generateActivityXml($activity, $transaction, $result, $settings, $this->xmlGenerator->generateActivityXml($activity, $transaction, $result, $settings, $organization); } + /** + * Generates combines activities xml file and publishes to IATI. + * + * @param $activities + * @param $settings + * @param $organization + * + * @return void + */ + public function generateActivitiesXml($activities, $settings, $organization): void + { + $this->xmlGenerator->generateActivitiesXml($activities, $settings, $organization); + } + /** * Deletes the unpublished file from server. * diff --git a/app/IATI/Traits/XmlBaseElement.php b/app/IATI/Traits/XmlBaseElement.php index 2907dbfb76..c993a3718d 100644 --- a/app/IATI/Traits/XmlBaseElement.php +++ b/app/IATI/Traits/XmlBaseElement.php @@ -23,21 +23,60 @@ public function buildNarrative($narratives): array $narrativeData = []; if ($narratives) { - foreach ($narratives as $narrative) { - if ($narrative != '') { - $narrativeData[] = [ - '@value' => Arr::get($narrative, 'narrative', null), - '@attributes' => [ - 'xml:lang' => Arr::get($narrative, 'language', null), - ], - ]; - } - } + $narrativeData = iterator_to_array($this->getNarrative($narratives)); + // foreach ($narratives as $narrative) { + // if ($narrative != '') { + // $narrativeData[] = [ + // '@value' => Arr::get($narrative, 'narrative', null), + // '@attributes' => [ + // 'xml:lang' => Arr::get($narrative, 'language', null), + // ], + // ]; + // } + // } } return $narrativeData; } + protected function getNarrative($narratives) + { + foreach ($narratives as $narrative) { + // if ($narrative != '') { + $narrativeData = [ + '@value' => Arr::get($narrative, 'narrative', null), + '@attributes' => [ + 'xml:lang' => Arr::get($narrative, 'language', null), + ], + ]; + + yield $narrativeData; + // } + } + } + + protected function category($categories) + { + foreach ($categories as $value) { + $category = [ + '@attributes' => ['code' => Arr::get($value, 'code', null)], + ]; + + yield $category; + } + } + + protected function language($languages) + { + foreach ($languages as $value) { + $language = [ + '@attributes' => ['code' => Arr::get($value, 'language', null)], + ]; + + yield $language; + } + } + /** * Returns xml data for document link. * @@ -51,21 +90,21 @@ protected function buildDocumentLink($documentLinks): array if (count($documentLinks)) { foreach ($documentLinks as $documentLink) { - $categories = []; + // $categories = []; - foreach (Arr::get($documentLink, 'category', []) as $value) { - $categories[] = [ - '@attributes' => ['code' => Arr::get($value, 'code', null)], - ]; - } + // foreach (Arr::get($documentLink, 'category', []) as $value) { + // $categories[] =[ + // '@attributes' => ['code' => Arr::get($value, 'code', null)], + // ]; + // } - $languages = []; + // $languages = []; - foreach (Arr::get($documentLink, 'language', []) as $language) { - $languages[] = [ - '@attributes' => ['code' => Arr::get($language, 'language', null)], - ]; - } + // foreach (Arr::get($documentLink, 'language', []) as $language) { + // $languages[] = [ + // '@attributes' => ['code' => Arr::get($language, 'language', null)], + // ]; + // } $documentLinkData[] = [ '@attributes' => [ @@ -78,8 +117,8 @@ protected function buildDocumentLink($documentLinks): array 'description' => [ 'narrative' => $this->buildNarrative(Arr::get($documentLink, 'description.0.narrative', [])), ], - 'category' => $categories, - 'language' => $languages, + 'category' => iterator_to_array($this->category(Arr::get($documentLink, 'category', []))), + 'language' => iterator_to_array($this->category(Arr::get($documentLink, 'language', []))), 'document-date' => [ '@attributes' => [ 'iso-date' => Arr::get($documentLink, 'document_date.0.date', null), diff --git a/app/Jobs/BulkPublishActivities.php b/app/Jobs/BulkPublishActivities.php index eca2644a2f..0c5675b470 100644 --- a/app/Jobs/BulkPublishActivities.php +++ b/app/Jobs/BulkPublishActivities.php @@ -26,6 +26,16 @@ class BulkPublishActivities implements ShouldQueue */ protected object $activities; + /** + * @var object + */ + protected object $organization; + + /** + * @var object + */ + protected object $settings; + /** * @var int */ @@ -55,14 +65,18 @@ class BulkPublishActivities implements ShouldQueue * Create a new job instance. * * @param $activities + * @param $organization + * @param $settings * @param $organizationId * @param $uuid * * @return void */ - public function __construct($activities, $organizationId, $uuid) + public function __construct($activities, $organization, $settings, $organizationId, $uuid) { $this->activities = $activities; + $this->organization = $organization; + $this->settings = $settings; $this->organizationId = $organizationId; $this->uuid = $uuid; } @@ -89,13 +103,16 @@ public function handle(BulkPublishingStatusService $publishingStatusService, Act $publishFile = true; } - if ($this->publishingStatusService->updateActivityStatus($activity->id, $this->uuid, 'processing')) { - $this->publishActivity($activity, $publishFile); - } - + $this->publishingStatusService->updateActivityStatus($activity->id, $this->uuid, 'processing'); $counter++; } + + logger()->error('publishing activities'); + logger()->error((string) count($this->activities)); + $this->publishActivity($this->activities, $this->organization, $this->settings, $publishFile); } + logger()->error('bulk publishing completed'); + logger()->error('-------------------------------'); } /** @@ -118,19 +135,27 @@ public function setServices($publishingStatusService, $activityWorkflowService, * Publishes activity and updates publish status table. * * @param $activity + * @param $organization + * @param $settings * @param $publishFile * * @return void */ - public function publishActivity($activity, $publishFile): void + public function publishActivity($activities, $organization, $settings, $publishFile): void { try { - $this->activityWorkflowService->publishActivity($activity, $publishFile); - $this->publishingStatusService->updateActivityStatus($activity->id, $this->uuid, 'completed'); + $this->activityWorkflowService->publishActivities($activities, $organization, $settings, $publishFile); + logger()->error('update activity status'); + foreach ($activities as $activity) { + $this->publishingStatusService->updateActivityStatus($activity->id, $this->uuid, 'completed'); + } } catch (\Exception $e) { + logger()->error($e); awsUploadFile('error-bulk-publish.log', $e->getMessage()); - $this->activityService->updatePublishedStatus($activity, 'draft', false); - $this->publishingStatusService->updateActivityStatus($activity->id, $this->uuid, 'failed'); + foreach ($activities as $activity) { + $this->activityService->updatePublishedStatus($activity, 'draft', false); + $this->publishingStatusService->updateActivityStatus($activity->id, $this->uuid, 'failed'); + } } } @@ -143,7 +168,7 @@ public function failed(): void { try { app(BulkPublishingStatusRepository::class)->failStuckActivities($this->organizationId); - } catch(\Exception $e) { + } catch (\Exception $e) { awsUploadFile('error-bulk-publish.log', $e->getMessage()); } } diff --git a/app/Providers/ExtendedMailServiceProvider.php b/app/Providers/ExtendedMailServiceProvider.php index 95c3a70dd7..107e64e7e4 100644 --- a/app/Providers/ExtendedMailServiceProvider.php +++ b/app/Providers/ExtendedMailServiceProvider.php @@ -20,8 +20,8 @@ class ExtendedMailServiceProvider extends MailServiceProvider */ public function boot(): void { - if (!App::environment(['production', 'staging'])) { - Mail::fake(); - } + // if (!App::environment(['production', 'staging'])) { + // Mail::fake(); + // } } }