Skip to content

Commit

Permalink
Merge branch '1269-delay-in-republishing' into develop
Browse files Browse the repository at this point in the history
  • Loading branch information
@sriza
sriza committed Jul 21, 2023
2 parents bf286c1 + f386b3b commit 75d0601
Show file tree
Hide file tree
Showing 9 changed files with 253 additions and 53 deletions.
21 changes: 17 additions & 4 deletions app/Http/Controllers/Admin/Workflow/BulkPublishingController.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,7 @@ public function checkCoreElementsCompleted(Request $request): JsonResponse
return response()->json(['success' => false, 'message' => trans('responses.no_activities_selected')]);
} catch (\Exception $e) {
logger()->error($e->getMessage());
logger()->error($e);

return response()->json(['success' => false, 'message' => trans('responses.error_has_occurred', ['event'=>trans('events.checking'), 'suffix'=>trans('common.core_completed_title')])]);
}
Expand Down Expand Up @@ -153,6 +154,7 @@ public function validateActivities(Request $request): JsonResponse
} catch (\Exception $e) {
DB::rollBack();
logger()->error($e->getMessage());
logger()->error($e);

return response()->json(['success' => false, 'message' => trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('elements_common.activities')])]);
}
Expand All @@ -169,18 +171,22 @@ public function validateActivities(Request $request): JsonResponse
*/
public function startBulkPublish(Request $request): JsonResponse
{
logger()->error('starting bulk publish entered');
try {
DB::beginTransaction();
$organization = auth()->user()->organization;

$message = $this->activityWorkflowService->getPublishErrorMessage(auth()->user()->organization);
$message = $this->activityWorkflowService->getPublishErrorMessage($organization);

if (!empty($message)) {
Session::put('error', $message);

return response()->json(['success' => false, 'message' => $message]);
}

if ($this->publishingStatusService->ongoingBulkPublishing(auth()->user()->organization->id)) {
logger()->error('checking bulk publishing is ongoing');

if ($this->publishingStatusService->ongoingBulkPublishing($organization->id)) {
$pubishingStatus = $this->bulkPublishingService->getOrganisationBulkPublishingStatus();

return response()->json([
Expand All @@ -194,15 +200,18 @@ public function startBulkPublish(Request $request): JsonResponse
$activityIds = json_decode($request->get('activities'), false, 512, JSON_THROW_ON_ERROR);

if (!empty($activityIds)) {
logger()->error('fetching activities');
$activities = $this->activityService->getActivitiesHavingIds($activityIds);

if (!count($activities)) {
return response()->json(['success' => false, 'message' => trans('responses.no_activities_selected')]);
}

logger()->error('storing initial response ');
$response = $this->bulkPublishingService->generateInitialBulkPublishingResponse($activities);
$this->publishingStatusService->storeProcessingActivities($activities, $response['organization_id'], $response['job_batch_uuid']);
dispatch(new BulkPublishActivities($activities, $response['organization_id'], $response['job_batch_uuid']));
logger()->error('start bulk publishing');
dispatch(new BulkPublishActivities($activities, $organization, $organization->settings, $response['organization_id'], $response['job_batch_uuid']));
logger()->error('bulk publishing completed');
DB::commit();

return response()->json(['success' => true, 'message' => trans('responses.bulk_publishing', ['suffix'=>trans('responses.started')]), 'data' => $response]);
Expand All @@ -212,6 +221,7 @@ public function startBulkPublish(Request $request): JsonResponse
} catch (\Exception $e) {
DB::rollBack();
logger()->error($e->getMessage());
logger()->error($e);

return response()->json(['success' => false, 'message' => trans('responses.bulk_publishing', ['suffix'=>trans('responses.failed')])]);
}
Expand Down Expand Up @@ -280,6 +290,7 @@ public function cancelBulkPublishing(): JsonResponse
} catch (\Exception $e) {
DB::rollBack();
logger()->error($e->getMessage());
logger()->error($e);

return response()->json(['success' => false, 'message' => trans('responses.no_bulk_publish_were_cancelled')]);
}
Expand Down Expand Up @@ -313,6 +324,7 @@ public function checksForActivityBulkPublish(): JsonResponse
return response()->json(['success' => true, 'message' => trans('responses.activity_ready_to_publish'), 'status' => 'completed']);
} catch (\Exception $e) {
logger()->error($e->getMessage());
logger()->error($e);

return response()->json(['success' => false, 'message' => trans('responses.error_has_occurred', ['event'=>trans('events.checking'), 'suffix'=>trans('common.activity')])]);
}
Expand All @@ -334,6 +346,7 @@ public function clearBulkPublishStatus(): JsonResponse
return response()->json(['success' => true, 'message' => 'Bulk publishing status successfully deleted.']);
} catch (\Exception $e) {
logger()->error($e->getMessage());
logger()->error($e);
DB::rollBack();

return response()->json(['success' => false, 'message' => 'Failed to delete bulk publishing status']);
Expand Down
70 changes: 62 additions & 8 deletions app/IATI/Elements/Xml/XmlGenerator.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ public function __construct(
}

/**
* Generates combines activities xml file and publishes to IATI.
* Generates single activity and combines xml file and publishes to IATI.
*
* @param $activity
* @param $transaction
Expand All @@ -265,7 +265,52 @@ public function generateActivityXml($activity, $transaction, $result, $settings,

if ($result) {
$publishedFiles = $this->savePublishedFiles($filename, $activity->org_id, $publishedActivity);
logger()->error('saving published file');
$this->getMergeXml($publishedFiles, $filename);
logger()->error('merged activity');
}
}

/**
* Generates multiple activities and combines xml file and publishes to IATI.
*
* @param $activity
* @param $transaction
* @param $result
* @param $settings
* @param $organization
*
* @return void
*/
public function generateActivitiesXml($activityData, $settings, $organization)
{
$publishedFiles = [];
$publishingInfo = $settings->publishing_info;
$publisherId = Arr::get($publishingInfo, 'publisher_id', 'Not Available');
$filename = sprintf('%s-%s.xml', $publisherId, 'activities');

foreach ($activityData as $activity) {
$publishedActivity = sprintf('%s-%s.xml', $publisherId, $activity->id);
$xml = $this->getXml($activity, $activity->transaction ?? [], $activity->results ?? [], $settings, $organization);
$result = awsUploadFile(
sprintf('%s/%s/%s', 'xml', 'activityXmlFiles', $publishedActivity),
$xml->saveXML()
);

$publishedFiles[] = $publishedActivity;

// $publishedFiles = $this->savePublishedFiles($filename, $activity->org_id, $publishedActivity);
}

if (count($publishedFiles)) {
// foreach ($publishedFiles as $file) {
$filesToMerge = $this->savePublishedFiles($filename, $activity->org_id, $publishedFiles);
// }
// dd($filesToMerge);

logger()->error('saving published file');
$this->getMergeXml($filesToMerge, $filename);
logger()->error('merged activity');
}
}

Expand All @@ -282,15 +327,21 @@ public function savePublishedFiles($filename, $organizationId, $publishedActivit
{
$published = $this->activityPublishedService->findOrCreate($filename, $organizationId);
$publishedActivities = $publishedActivity;
// dd($publishedActivities, $published->published_activities);

if (!is_array($publishedActivity)) {
$publishedActivities = (array) $published->published_activities;
(in_array($publishedActivity, $publishedActivities, true)) ?: array_push($publishedActivities, $publishedActivity);
}

if (is_array($publishedActivity)) {
$publishedActivities = (array) $published->published_activities;
$publishedActivities = array_merge($publishedActivities, $publishedActivity);
}

$this->activityPublishedService->update($published, array_unique($publishedActivities));

return $published->published_activities;
return $publishedActivities;
}

/**
Expand All @@ -312,6 +363,7 @@ public function getMergeXml($publishedFiles, $filename): void
$iatiActivities->appendChild($dom->createComment('Generated By IATI Publisher'));

foreach ($publishedFiles as $xml) {
logger()->error('xml');
$addDom = new \DOMDocument();
$fileContent = awsGetFile(sprintf('%s/%s/%s', 'xml', 'activityXmlFiles', $xml));

Expand Down Expand Up @@ -392,12 +444,12 @@ public function getXmlAttributes($defaultValues): array
{
$data = [
'last-updated-datetime' => gmdate('c', time()),
'xml:lang' => Arr::get($defaultValues, 'default_language', null),
'default-currency' => Arr::get($defaultValues, 'default_currency', null),
'humanitarian' => Arr::get($defaultValues, 'humanitarian', 1),
'hierarchy' => Arr::get($defaultValues, 'hierarchy', 1),
'budget-not-provided' => Arr::get($defaultValues, 'budget_not_provided', ''),
'linked-data-uri' => Arr::get($defaultValues, 'linked_data_uri', ''),
'xml:lang' => Arr::get($defaultValues, 'default_language', null),
'default-currency' => Arr::get($defaultValues, 'default_currency', null),
'humanitarian' => Arr::get($defaultValues, 'humanitarian', 1),
'hierarchy' => Arr::get($defaultValues, 'hierarchy', 1),
'budget-not-provided' => Arr::get($defaultValues, 'budget_not_provided', ''),
'linked-data-uri' => Arr::get($defaultValues, 'linked_data_uri', ''),
];

foreach ($data as $key => $datum) {
Expand Down Expand Up @@ -463,6 +515,7 @@ public function setServices(): void
*/
public function getXmlData($activity, $transaction, $result, $organization): array
{
logger()->error('starting xml data fetching');
$xmlActivity = [];
$xmlActivity['iati-identifier'] = ($organization->identifier ?: 'Not Available') . '-' . Arr::get($activity->iati_identifier, 'activity_identifier', 'Not Available');
$xmlActivity['reporting-org'] = $this->reportingOrgService->getXmlData($activity);
Expand Down Expand Up @@ -499,6 +552,7 @@ public function getXmlData($activity, $transaction, $result, $organization): arr
removeEmptyValues($xmlActivity);

$this->mapActivityTransactionAndResultIndex($xmlActivity, $activity);
logger()->error('xml data fetched');

return $xmlActivity;
}
Expand Down
2 changes: 2 additions & 0 deletions app/IATI/Services/Publisher/PublisherService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,11 +52,13 @@ public function __construct(ActivityPublishedService $activityPublishedService,
*/
public function publishFile($registryInfo, $activityPublished, $organization, bool $updatedActivityPublished = true): void
{
logger()->error('publishing file stated');
$this->setFile($activityPublished);
$this->init(env('IATI_API_ENDPOINT'), Arr::get($registryInfo, 'api_token', ''))
->setPublisher(Arr::get($registryInfo, 'publisher_id', ''));
$this->searchForPublisher($this->publisherId);
$this->publishToRegistry($organization, $activityPublished->filename, $updatedActivityPublished);
logger()->error('file published');
}

/**
Expand Down
44 changes: 43 additions & 1 deletion app/IATI/Services/Workflow/ActivityWorkflowService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,7 @@ public function findActivity($activityId): object
*/
public function publishActivity($activity, bool $publishFile = true): void
{
logger()->error('publishing activity');
$organization = $activity->organization;
$settings = $organization->settings;
$this->xmlGeneratorService->generateActivityXml(
Expand All @@ -135,15 +136,53 @@ public function publishActivity($activity, bool $publishFile = true): void
$settings,
$organization
);
logger()->error('activity xml generation');

if ($publishFile) {
logger()->error('publising xml file');
$activityPublished = $this->activityPublishedService->getActivityPublished($organization->id);
$publishingInfo = $settings->publishing_info;
$this->publisherService->publishFile($publishingInfo, $activityPublished, $organization);
logger()->error('xml file published');
}

$this->activityService->updatePublishedStatus($activity, 'published', true);
logger()->error('updated publisher status');
$this->activitySnapshotService->createOrUpdateActivitySnapshot($activity);
logger()->error('updated snapshot');
}

/**
* Publish an activities to the IATI registry.
*
* @param $activities
* @param $organization
* @param $settings
* @param bool $publishFile
*
* @return void
*/
public function publishActivities($activities, $organization, $settings, bool $publishFile = true): void
{
logger()->error('activity xml generation');
$this->xmlGeneratorService->generateActivitiesXml(
$activities,
$settings,
$organization
);

logger()->error('publising xml file');
$activityPublished = $this->activityPublishedService->getActivityPublished($organization->id);
$publishingInfo = $settings->publishing_info;
$this->publisherService->publishFile($publishingInfo, $activityPublished, $organization);
logger()->error('xml file published');

foreach ($activities as $activity) {
$this->activityService->updatePublishedStatus($activity, 'published', true);
logger()->error('updated publisher status');
$this->activitySnapshotService->createOrUpdateActivitySnapshot($activity);
logger()->error('updated snapshot');
}
}

/**
Expand Down Expand Up @@ -202,6 +241,7 @@ public function validateActivityOnIATIValidator($activity): string
$settings,
$organization
);
logger()->error("xmlValidation/$activity->org_id/activity_$activity->id.xml created");

awsUploadFile("xmlValidation/$activity->org_id/activity_$activity->id.xml", $xmlData);

Expand All @@ -220,6 +260,7 @@ public function validateActivityOnIATIValidator($activity): string
*/
public function getResponse($xmlData): string
{
file_put_contents(storage_path('test.xml'), $xmlData);
$client = new Client();
$URI = env('IATI_VALIDATOR_ENDPOINT');
$params['headers'] = ['Content-Type' => 'application/json', 'Ocp-Apim-Subscription-Key' => env('IATI_VALIDATOR_KEY')];
Expand Down Expand Up @@ -340,7 +381,8 @@ public function populateSectorIfMissing($activity): object
{
$data = sectorDefaultValue();

if ((empty($activity->sector) && !$this->activityService->checkIfTransactionHasSector($activity))
if (
(empty($activity->sector) && !$this->activityService->checkIfTransactionHasSector($activity))
|| (is_variable_null($activity->sector))
) {
$this->sectorService->update($activity->id, $data);
Expand Down
15 changes: 13 additions & 2 deletions app/IATI/Services/Workflow/BulkPublishingService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,7 @@ public function validateActivitiesOnIATI($activityIds): array
if ($activity && $activity->status === 'draft') {
$response = $this->validateWithException($activity);
$this->apiLogRepo->store(generateApiInfo('POST', env('IATI_VALIDATOR_ENDPOINT'), ['form_params' => json_encode($activity)], json_encode($response)));
logger()->error('activity validated');

if (!Arr::get($response, 'success', true)) {
logger()->error(trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('responses.activity_with_id')]) . $activityId);
Expand Down Expand Up @@ -160,28 +161,38 @@ public function validateActivitiesOnIATI($activityIds): array
*/
public function validateWithException($activity): array
{
logger()->error('validating with exception');
try {
$validatorResponse = $this->activityWorkflowService->validateActivityOnIATIValidator($activity);
logger()->error('validator response');
// $response = $validatorResponse;
$response = $this->addElementOnIatiValidatorResponse($validatorResponse, $activity);

logger()->error('validator response attested');
if ($this->validatorService->updateOrCreateResponse($activity->id, $response)) {
return $response;
}
logger()->error('validator response saved');

return ['success' => false, 'error' => trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('elements_common.activity')])];
} catch (BadResponseException $ex) {
logger()->error('client error');
if ($ex->getCode() === 422) {
$validatorResponse = $ex->getResponse()->getBody()->getContents();
$response = $this->addElementOnIatiValidatorResponse($validatorResponse, $activity);
$response = $validatorResponse;
$response = json_decode($validatorResponse, true, 512, JSON_THROW_ON_ERROR);

// $response = $this->addElementOnIatiValidatorResponse($validatorResponse, $activity);

if ($this->validatorService->updateOrCreateResponse($activity->id, $response)) {
return $response;
}
}
logger()->error('completed');

return ['success' => false, 'error' => trans('responses.error_has_occurred', ['event'=>trans('events.validating'), 'suffix'=>trans('elements_common.activity')])];
} catch (\Exception $e) {
logger()->error($e->getMessage());
logger()->error($e);

throw new \RuntimeException();
}
Expand Down
14 changes: 14 additions & 0 deletions app/IATI/Services/Xml/XmlGeneratorService.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,20 @@ public function generateActivityXml($activity, $transaction, $result, $settings,
$this->xmlGenerator->generateActivityXml($activity, $transaction, $result, $settings, $organization);
}

/**
* Generates combines activities xml file and publishes to IATI.
*
* @param $activities
* @param $settings
* @param $organization
*
* @return void
*/
public function generateActivitiesXml($activities, $settings, $organization): void
{
$this->xmlGenerator->generateActivitiesXml($activities, $settings, $organization);
}

/**
* Deletes the unpublished file from server.
*
Expand Down
Loading

0 comments on commit 75d0601

Please sign in to comment.