Skip to content

Commit

Permalink
Merge pull request #473 from bcgov/develop/alex-GRAD2-2377
Browse files Browse the repository at this point in the history 
Develop/alex grad2 2377
  • Loading branch information
@githubmamatha
githubmamatha authored Apr 12, 2024
2 parents ed3e9c5 + 24f0912 commit c870ce2
Show file tree
Hide file tree
Showing 28 changed files with 136 additions and 118 deletions.
5 changes: 2 additions & 3 deletions ...uc/api/batchgraduation/listener/UserReqDistributionRunCompletionNotificationListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -164,7 +164,7 @@ private void addTranscriptsToDistributionRequest(List<StudentCredentialDistribut
String studentPensSearchRequest = jsonTransformer.marshall(searchRequest);
LOGGER.debug("Get {} students credentials for the pens: {}", "OT", studentPensSearchRequest);
}
List<StudentCredentialDistribution> transcriptDistributionList = restUtils.getStudentsForUserReqDisRun("OT",searchRequest,restUtils.getTokenResponseObject().getAccess_token());
List<StudentCredentialDistribution> transcriptDistributionList = restUtils.getStudentsForUserReqDisRun("OT",searchRequest);
for(StudentCredentialDistribution certScd: uniqueCertificateList) {
for(StudentCredentialDistribution trScd: transcriptDistributionList) {
if(certScd.getStudentID().equals(trScd.getStudentID())) {
Expand Down Expand Up @@ -196,8 +196,7 @@ private void updateBackStudentRecords(List<StudentCredentialDistribution> cList,
});
List<UUID> studentIDs = cList.stream().map(StudentCredentialDistribution::getStudentID).distinct().collect(Collectors.toList());
studentIDs.forEach(sid-> {
String accessToken = restUtils.fetchAccessToken();
restUtils.updateStudentGradRecord(sid,batchId,activityCode,accessToken);
restUtils.updateStudentGradRecord(sid,batchId,activityCode);
});
}

Expand Down
2 changes: 1 addition & 1 deletion ...api/batchgraduation/listener/UserReqPsiDistributionRunCompletionNotificationListener.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private void updateBackStudentRecords(List<PsiCredentialDistribution> cList, Lon
cList.forEach(scd-> {
LOGGER.debug("Update back Student Record {}", scd.getStudentID());
String accessToken = restUtils.fetchAccessToken();
restUtils.updateStudentGradRecord(scd.getStudentID(),batchId,activityCode,accessToken);
restUtils.updateStudentGradRecord(scd.getStudentID(),batchId,activityCode);
});
}
}
1 change: 1 addition & 0 deletions ...in/java/ca/bc/gov/educ/api/batchgraduation/model/GraduationStudentRecordDistribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class GraduationStudentRecordDistribution extends BaseModel{
private String legalMiddleNames;
private String legalLastName;
private String schoolOfRecord;
private String schoolAtGrad;
private String programCompletionDate;
private String honoursStanding;
private String program;
Expand Down
2 changes: 1 addition & 1 deletion ...va/ca/bc/gov/educ/api/batchgraduation/processor/DistributionRunStatusUpdateProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ private Map<String, ServiceException> updateBackStudentRecords(List<StudentCrede
"NONGRADYERUN".equalsIgnoreCase(activityCode)? "IP" : scd.getDocumentStatusCode(),activityCode,token);
LOGGER.debug("Dist Job [{}] / [{}] - update {} of {} student credential record: studentID, credentials, document status [{}, {}, {}]", batchId, activityCode, processedCount[0] + 1, totalCount, scd.getStudentID(), scd.getCredentialTypeCode(), scd.getDocumentStatusCode());
if(!StringUtils.equalsAnyIgnoreCase(jobType, "DISTRUN_YE", "NONGRADRUN", "REGALG", "TVRRUN")) {
restUtils.updateStudentGradRecord(scd.getStudentID(), batchId, activityCode, token);
restUtils.updateStudentGradRecord(scd.getStudentID(), batchId, activityCode);
LOGGER.debug("Dist Job [{}] / [{}] - update {} of {} student grad record: studentID, credentials, document status [{}, {}, {}]", batchId, activityCode, processedCount[0] + 1, totalCount, scd.getStudentID(), scd.getCredentialTypeCode(), scd.getDocumentStatusCode());
}
processedCount[0]++;
Expand Down
3 changes: 1 addition & 2 deletions ...rc/main/java/ca/bc/gov/educ/api/batchgraduation/processor/EDWSnapshotSchoolProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import org.springframework.beans.factory.annotation.Value;

import java.util.List;
import java.util.stream.Collectors;

public class EDWSnapshotSchoolProcessor implements ItemProcessor<String, List<Pair<String, List<SnapshotResponse>>>> {

Expand All @@ -31,7 +30,7 @@ public class EDWSnapshotSchoolProcessor implements ItemProcessor<String, List<Pa
public List<Pair<String, List<SnapshotResponse>>> process(String mincode) throws Exception {
summaryDTO.setBatchId(batchId);
LOGGER.debug("Processing partitionData for School: {} ", mincode);
List<SnapshotResponse> edwStudents = restUtils.getEDWSnapshotStudents(summaryDTO.getGradYear(), mincode, summaryDTO.getAccessToken());
List<SnapshotResponse> edwStudents = restUtils.getEDWSnapshotStudents(summaryDTO.getGradYear(), mincode);
if ("L".equals(summaryDTO.getOption())) {
List<SnapshotResponse> list = edwStudents.stream().filter(s -> StringUtils.isNotBlank(s.getGraduatedDate()) ||
("12".equals(s.getStudentGrade()) || "AD".equals(s.getStudentGrade()))).toList();
Expand Down
6 changes: 3 additions & 3 deletions ...ava/ca/bc/gov/educ/api/batchgraduation/processor/RunCertificateRegenerationProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ public class RunCertificateRegenerationProcessor implements ItemProcessor<UUID,

@Override
public Integer process(UUID key) throws Exception {
GraduationStudentRecordDistribution stuRec = restUtils.getStudentData(key.toString(), summaryDTO.getAccessToken());
GraduationStudentRecordDistribution stuRec = restUtils.getStudentData(key.toString());
if (stuRec != null) {
LOGGER.info("Processing partitionData: studentID = {}", stuRec.getStudentID());
summaryDTO.setBatchId(batchId);
try {
summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() + 1L);
Integer count = restUtils.runRegenerateStudentCertificate(stuRec.getPen(), summaryDTO.getAccessToken());
Integer count = restUtils.runRegenerateStudentCertificate(stuRec.getPen());
if (count > 0) {
restUtils.updateStudentGradRecord(key, batchId, "CERTREGEN", summaryDTO.getAccessToken());
restUtils.updateStudentGradRecord(key, batchId, "CERTREGEN");
}
return count;
} catch(Exception e) {
Expand Down
2 changes: 1 addition & 1 deletion api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/BasePartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,7 @@ void filterByStudentSearchRequest(List<StudentCredentialDistribution> eligibleSt

void filterOutDeceasedStudents(List<StudentCredentialDistribution> credentialList) {
LOGGER.debug("Total size of credential list: {}", credentialList.size());
List<UUID> deceasedIDs = restUtils.getDeceasedStudentIDs(credentialList.stream().map(StudentCredentialDistribution::getStudentID).distinct().toList(), restUtils.getAccessToken());
List<UUID> deceasedIDs = restUtils.getDeceasedStudentIDs(credentialList.stream().map(StudentCredentialDistribution::getStudentID).distinct().toList());
if (!deceasedIDs.isEmpty()) {
LOGGER.debug("Deceased students: {}", deceasedIDs.size());
credentialList.removeIf(cr -> deceasedIDs.contains(cr.getStudentID()));
Expand Down
5 changes: 2 additions & 3 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ public Map<String, ExecutionContext> partition(int gridSize) {
// Clean up existing reports before running new one
logger.debug("Delete School Reports for Monthly Distribution");
long startTime = System.currentTimeMillis();
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL");
long endTime = System.currentTimeMillis();
long diff = (endTime - startTime)/1000;
logger.debug("Old School Reports deleted in {} sec", diff);

startTime = System.currentTimeMillis();
logger.debug("Retrieve students for Monthly Distribution");
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData(restUtils.getAccessToken());
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData();
DistributionDataParallelDTO parallelDTO = parallelDTOMono.block();
List<StudentCredentialDistribution> credentialList = new ArrayList<>();
if(parallelDTO != null) {
Expand All @@ -62,7 +62,6 @@ public Map<String, ExecutionContext> partition(int gridSize) {
return new HashMap<>();
}


@Override
protected JobExecution getJobExecution() {
return context;
Expand Down
2 changes: 1 addition & 1 deletion .../java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitionerPsiUserReq.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {
String transmissionType = jobParameters.getString("transmissionType");
PsiCredentialRequest req = (PsiCredentialRequest)jsonTransformer.unmarshall(searchRequest, PsiCredentialRequest.class);
String accessToken = restUtils.getAccessToken();
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_PSI", accessToken);
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_PSI");

List<PsiCredentialDistribution> credentialList = getRecordsForPSIUserReqDisRun(req,transmissionType,accessToken);
if(!credentialList.isEmpty()) {
Expand Down
8 changes: 1 addition & 7 deletions ...ain/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunPartitionerUserReq.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package ca.bc.gov.educ.api.batchgraduation.reader;

import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest;
import org.slf4j.Logger;
Expand All @@ -23,15 +22,10 @@ public class DistributionRunPartitionerUserReq extends BasePartitioner {

@Override
public Map<String, ExecutionContext> partition(int gridSize) {
ResponseObj res = restUtils.getTokenResponseObject();
String accessToken = null;
if (res != null) {
accessToken = res.getAccess_token();
}
JobParameters jobParameters = context.getJobParameters();
String credentialType = jobParameters.getString("credentialType");
StudentSearchRequest req = getStudentSearchRequest();
List<StudentCredentialDistribution> credentialList = restUtils.getStudentsForUserReqDisRun(credentialType,req,accessToken);
List<StudentCredentialDistribution> credentialList = restUtils.getStudentsForUserReqDisRun(credentialType,req);
if(!credentialList.isEmpty()) {
Map<String, ExecutionContext> map = getStringExecutionContextMap(gridSize, credentialList, credentialType);
LOGGER.info("Found {} in total running on {} partitions",credentialList.size(),map.size());
Expand Down
10 changes: 5 additions & 5 deletions ...ava/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunSupplementalPartitioner.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package ca.bc.gov.educ.api.batchgraduation.reader;

import ca.bc.gov.educ.api.batchgraduation.model.DistributionDataParallelDTO;
import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
import ca.bc.gov.educ.api.batchgraduation.service.ParallelDataFetch;
import org.slf4j.Logger;
Expand All @@ -9,9 +8,10 @@
import org.springframework.batch.item.ExecutionContext;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import reactor.core.publisher.Mono;

import java.util.*;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class DistributionRunSupplementalPartitioner extends BasePartitioner {

Expand All @@ -29,8 +29,8 @@ public Map<String, ExecutionContext> partition(int gridSize) {
logger.debug("Delete School Reports for Supplemental Distribution");
long startTime = System.currentTimeMillis();
// Clean up existing reports before running new one
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "DISTREP_SC", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL");
restUtils.deleteSchoolReportRecord("", "DISTREP_SC");
long endTime = System.currentTimeMillis();
long diff = (endTime - startTime)/1000;
logger.debug("Old School Reports deleted in {} sec", diff);
Expand Down
10 changes: 5 additions & 5 deletions ...va/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunYearlyNonGradPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ public Map<String, ExecutionContext> partition(int gridSize) {
// Clean up existing reports before running new one
logger.debug("Delete School Reports for Yearly Distribution");
long startTime = System.currentTimeMillis();
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_YE", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "NONGRADDISTREP_SC", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "NONGRADDISTREP_SD", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL");
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_YE");
restUtils.deleteSchoolReportRecord("", "NONGRADDISTREP_SC");
restUtils.deleteSchoolReportRecord("", "NONGRADDISTREP_SD");
long endTime = System.currentTimeMillis();
long diff = (endTime - startTime)/1000;
logger.debug("Old School Reports deleted in {} sec", diff);
Expand All @@ -52,7 +52,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {
if(eligibleStudentSchoolDistricts.isEmpty() ) {
logger.debug("No filter found, retrieve all districts");
startTime = System.currentTimeMillis();
eligibleStudentSchoolDistricts = parallelDataFetch.fetchDistributionRequiredDataDistrictsNonGradYearly(restUtils.getAccessToken());
eligibleStudentSchoolDistricts = parallelDataFetch.fetchDistributionRequiredDataDistrictsNonGradYearly();
endTime = System.currentTimeMillis();
diff = (endTime - startTime)/1000;
logger.debug("All {} districts retrieved in {} sec", eligibleStudentSchoolDistricts.size(), diff);
Expand Down
8 changes: 4 additions & 4 deletions ...main/java/ca/bc/gov/educ/api/batchgraduation/reader/DistributionRunYearlyPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,10 +29,10 @@ public Map<String, ExecutionContext> partition(int gridSize) {
// Clean up existing reports before running new one
logger.debug("Delete School Reports for Yearly Distribution");
long startTime = System.currentTimeMillis();
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_YE", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "DISTREP_YE_SC", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "DISTREP_YE_SD", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL");
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_YE");
restUtils.deleteSchoolReportRecord("", "DISTREP_YE_SC");
restUtils.deleteSchoolReportRecord("", "DISTREP_YE_SD");
long endTime = System.currentTimeMillis();
long diff = (endTime - startTime)/1000;
logger.debug("Old School Reports deleted in {} sec", diff);
Expand Down
8 changes: 1 addition & 7 deletions ...src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/EDWSnapshotSchoolPartitioner.java
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package ca.bc.gov.educ.api.batchgraduation.reader;

import ca.bc.gov.educ.api.batchgraduation.model.EdwSnapshotSchoolSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.model.SnapshotRequest;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -26,11 +25,6 @@ public class EDWSnapshotSchoolPartitioner extends BasePartitioner {

@Override
public Map<String, ExecutionContext> partition(int gridSize) {
ResponseObj res = restUtils.getTokenResponseObject();
String accessToken = null;
if (res != null) {
accessToken = res.getAccess_token();
}
long startTime = System.currentTimeMillis();
logger.debug("Retrieve schools for EDW Snapshot");
JobParameters jobParameters = context.getJobParameters();
Expand All @@ -40,7 +34,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {
if (req.getSchoolOfRecords() != null && !req.getSchoolOfRecords().isEmpty()) {
schools = req.getSchoolOfRecords();
} else {
schools = restUtils.getEDWSnapshotSchools(req.getGradYear(), accessToken);
schools = restUtils.getEDWSnapshotSchools(req.getGradYear());
}
long endTime = System.currentTimeMillis();
long diff = (endTime - startTime)/1000;
Expand Down
8 changes: 1 addition & 7 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegGradAlgPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity;
import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.model.RunTypeEnum;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -34,12 +33,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {
BatchGradAlgorithmJobHistoryEntity jobHistory = createBatchJobHistory();
List<UUID> studentList;
if (runType == RunTypeEnum.NORMAL_JOB_PROCESS) {
ResponseObj res = restUtils.getTokenResponseObject();
String accessToken = null;
if (res != null) {
accessToken = res.getAccess_token();
}
studentList = restUtils.getStudentsForAlgorithm(accessToken);
studentList = restUtils.getStudentsForAlgorithm();
} else {
studentList = getInputDataFromPreviousJob();
}
Expand Down
6 changes: 3 additions & 3 deletions ...main/java/ca/bc/gov/educ/api/batchgraduation/reader/RegenerateCertificatePartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {
JobParameters jobParameters = context.getJobParameters();
String searchRequest = jobParameters.getString(SEARCH_REQUEST);
if (StringUtils.isBlank(searchRequest)) {
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData(accessToken);
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData();
DistributionDataParallelDTO parallelDTO = parallelDTOMono.block();
if(parallelDTO != null) {
credentialList.addAll(parallelDTO.certificateList());
Expand Down Expand Up @@ -84,7 +84,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {

private List<StudentCredentialDistribution> getStudentsForDistRun(String accessToken) {
List<StudentCredentialDistribution> credentialList = new ArrayList<>();
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData(accessToken);
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData();
DistributionDataParallelDTO parallelDTO = parallelDTOMono.block();
if(parallelDTO != null) {
credentialList.addAll(parallelDTO.certificateList());
Expand All @@ -103,7 +103,7 @@ private List<StudentCredentialDistribution> getStudentsForUserReqRun(String sear
if (certRegenReq != null && "Y".equalsIgnoreCase(certRegenReq.getRunMode())) {
StudentSearchRequest req = new StudentSearchRequest();
req.setPens(certRegenReq.getPens());
return restUtils.getStudentsForUserReqDisRun("OC", req, accessToken);
return restUtils.getStudentsForUserReqDisRun("OC", req);
} else {
return getStudentsForDistRun(accessToken);
}
Expand Down
8 changes: 1 addition & 7 deletions api/src/main/java/ca/bc/gov/educ/api/batchgraduation/reader/SpcRegGradAlgPartitioner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@

import ca.bc.gov.educ.api.batchgraduation.entity.BatchGradAlgorithmJobHistoryEntity;
import ca.bc.gov.educ.api.batchgraduation.model.AlgorithmSummaryDTO;
import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.model.RunTypeEnum;
import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest;
import org.slf4j.Logger;
Expand Down Expand Up @@ -38,15 +37,10 @@ public Map<String, ExecutionContext> partition(int gridSize) {
BatchGradAlgorithmJobHistoryEntity jobHistory = createBatchJobHistory();
List<UUID> studentList;
if (runType == RunTypeEnum.NORMAL_JOB_PROCESS) {
ResponseObj res = restUtils.getTokenResponseObject();
String accessToken = null;
if (res != null) {
accessToken = res.getAccess_token();
}
JobParameters jobParameters = jobExecution.getJobParameters();
String searchRequest = jobParameters.getString(SEARCH_REQUEST, "{}");
StudentSearchRequest req = (StudentSearchRequest)jsonTransformer.unmarshall(searchRequest, StudentSearchRequest.class);
studentList = restUtils.getStudentsForSpecialGradRun(req, accessToken);
studentList = restUtils.getStudentsForSpecialGradRun(req);
} else {
studentList = getInputDataFromPreviousJob();
}
Expand Down
Loading

0 comments on commit c870ce2

Please sign in to comment.