Skip to content

Commit

Permalink
Merge pull request #485 from bcgov/grad-release
Browse files Browse the repository at this point in the history
Grad release 1.18.0
  • Loading branch information
githubmamatha authored Apr 24, 2024
2 parents 3ba16a9 + b95c282 commit a679055
Show file tree
Hide file tree
Showing 44 changed files with 252 additions and 181 deletions.
2 changes: 1 addition & 1 deletion api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>ca.bc.gov.educ</groupId>
<artifactId>educ-grad-batch-graduation-api</artifactId>
<version>1.8.60</version>
<version>1.8.61</version>
<name>educ-grad-batch-graduation-api</name>
<description>Ministry of Education GRAD BATCH GRADUATION API</description>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -519,6 +519,12 @@ public DistributionRunProcessor itemProcessorDisRun() {
return new DistributionRunProcessor();
}

@Bean
@StepScope
public DistributionRunYearlyProcessor itemProcessorYearlyDisRun() {
return new DistributionRunYearlyProcessor();
}

@Bean
@StepScope
public DistributionRunYearlyNonGradProcessor itemProcessorDisRunYearlyNonGradByMincode() {
Expand Down Expand Up @@ -571,7 +577,7 @@ public Step slaveStepDisRunYearly(JobRepository jobRepository, PlatformTransacti
return new StepBuilder("slaveStepDisRun", jobRepository)
.<StudentCredentialDistribution, StudentCredentialDistribution>chunk(1, transactionManager)
.reader(itemReaderDisRun())
.processor(itemProcessorDisRun())
.processor(itemProcessorYearlyDisRun())
.writer(itemWriterDisRun())
.build();
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public WebClient webClient() {
return WebClient.builder().exchangeStrategies(ExchangeStrategies.builder()
.codecs(configurer -> configurer
.defaultCodecs()
.maxInMemorySize(300 * 1024 * 1024)) // 100 MB
.maxInMemorySize(300 * 1024 * 1024)) // 300 MB
.build()).build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
import lombok.Data;
import lombok.EqualsAndHashCode;

import java.time.Instant;
import java.time.LocalDateTime;

@Data
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,15 +128,15 @@ private void processGlobalList(DistributionSummaryDTO summaryDTO, Long batchId,
DistributionResponse disres = null;
String activityCode = null;
if(credentialType != null) {
if(credentialType.equalsIgnoreCase("OC")) {
if("OC".equalsIgnoreCase(credentialType)) {
activityCode = "USERDISTOC";
/** GRADT-553
* User Request Distribution Run - Original Certificate OC
**** Also select the students’ transcript for print
*/
addTranscriptsToDistributionRequest(cList,summaryDTO,batchId,properName);
} else {
activityCode = credentialType.equalsIgnoreCase("OT")?"USERDISTOT":"USERDISTRC";
activityCode = "OT".equalsIgnoreCase(credentialType) ? "USERDISTOT" : "USERDISTRC";
}
if(!cList.isEmpty()) {
DistributionRequest distributionRequest = DistributionRequest.builder().mapDist(mapDist).activityCode(activityCode).studentSearchRequest(summaryDTO.getStudentSearchRequest()).build();
Expand Down Expand Up @@ -164,7 +164,7 @@ private void addTranscriptsToDistributionRequest(List<StudentCredentialDistribut
String studentPensSearchRequest = jsonTransformer.marshall(searchRequest);
LOGGER.debug("Get {} students credentials for the pens: {}", "OT", studentPensSearchRequest);
}
List<StudentCredentialDistribution> transcriptDistributionList = restUtils.getStudentsForUserReqDisRun("OT",searchRequest,restUtils.getTokenResponseObject().getAccess_token());
List<StudentCredentialDistribution> transcriptDistributionList = restUtils.getStudentsForUserReqDisRun("OT",searchRequest);
for(StudentCredentialDistribution certScd: uniqueCertificateList) {
for(StudentCredentialDistribution trScd: transcriptDistributionList) {
if(certScd.getStudentID().equals(trScd.getStudentID())) {
Expand Down Expand Up @@ -196,8 +196,7 @@ private void updateBackStudentRecords(List<StudentCredentialDistribution> cList,
});
List<UUID> studentIDs = cList.stream().map(StudentCredentialDistribution::getStudentID).distinct().collect(Collectors.toList());
studentIDs.forEach(sid-> {
String accessToken = restUtils.fetchAccessToken();
restUtils.updateStudentGradRecord(sid,batchId,activityCode,accessToken);
restUtils.updateStudentGradRecord(sid,batchId,activityCode);
});
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ private void updateBackStudentRecords(List<PsiCredentialDistribution> cList, Lon
cList.forEach(scd-> {
LOGGER.debug("Update back Student Record {}", scd.getStudentID());
String accessToken = restUtils.fetchAccessToken();
restUtils.updateStudentGradRecord(scd.getStudentID(),batchId,activityCode,accessToken);
restUtils.updateStudentGradRecord(scd.getStudentID(),batchId,activityCode);
});
}
}
Original file line number Diff line number Diff line change
@@ -1,20 +1,22 @@
package ca.bc.gov.educ.api.batchgraduation.model;

import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.stereotype.Component;

import java.io.Serializable;
import java.util.List;

@Data
@Component
@NoArgsConstructor
@AllArgsConstructor
@Builder
public class CertificateRegenerationRequest implements Serializable {
private List<String> pens;
public class CertificateRegenerationRequest extends StudentSearchRequest {
private String runMode; // "Y" or "N"

public boolean runForAll () {
return (getPens() == null || getPens().isEmpty()) &&
(getDistricts() == null || getDistricts().isEmpty()) &&
(getSchoolOfRecords() == null || getSchoolOfRecords().isEmpty()) &&
(getSchoolCategoryCodes() == null || getSchoolCategoryCodes().isEmpty()) &&
(getStudentIDs() == null || getStudentIDs().isEmpty());
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public class GraduationStudentRecordDistribution extends BaseModel{
private String legalMiddleNames;
private String legalLastName;
private String schoolOfRecord;
private String schoolAtGrad;
private String programCompletionDate;
private String honoursStanding;
private String program;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ public enum JobProperName {
NDBJ("Non-Grad Distribution Batch Job"),
URDBJ ("User Req Distribution Batch Job"),
URPDBJ("PSI Distribution Batch Job"),
BDBJ ("Blank Distribution Batch Job");
BDBJ ("Blank Distribution Batch Job"),
RCBJ ("Regenerate Certificates Batch Job");

private final String value;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ public enum JobSelection {
NDBJ("NONGRADRUN"),
URDBJ ("DISTRUNUSER"),
URPDBJ ("PSIRUN"),
BDBJ ("DISTRUNUSER");
BDBJ ("DISTRUNUSER"),
RCBJ ("CERT_REGEN");

private final String value;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package ca.bc.gov.educ.api.batchgraduation.model;

import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Data;

import java.io.Serializable;
Expand Down Expand Up @@ -27,4 +28,9 @@ public class StudentCredentialDistribution implements Serializable {
private String studentGrade;
private List<GradRequirement> nonGradReasons;

@JsonIgnore
private String schoolAtGrad;
@JsonIgnore
private String schoolOfRecordOrigin;

}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import org.springframework.stereotype.Component;

import java.io.Serializable;
import java.util.Date;
import java.time.LocalDate;
import java.util.List;
import java.util.UUID;

Expand All @@ -29,9 +29,9 @@ public class StudentSearchRequest implements Serializable {
private Address address;

@JsonFormat(pattern = "yyyy-MM-dd")
Date gradDateFrom;
LocalDate gradDateFrom;
@JsonFormat(pattern = "yyyy-MM-dd")
Date gradDateTo;
LocalDate gradDateTo;

Boolean validateInput;
String localDownload;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,8 @@ public enum TaskSelection {
NDBJ ("YearlyNonGradDistributionBatchJob"),
URDBJ ("UserReqDistributionBatchJob"),
URPDBJ("psiDistributionBatchJob"),
BDBJ ("blankDistributionBatchJob");
BDBJ ("blankDistributionBatchJob"),
RCBJ ("certRegenBatchJob");

private final String value;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class DistributionRunProcessor implements ItemProcessor<StudentCredential
public StudentCredentialDistribution process(StudentCredentialDistribution item) throws Exception {
LOGGER.info("Processing partitionData = {}", item.getCredentialTypeCode());
summaryDTO.setBatchId(batchId);
return restUtils.processDistribution(item, summaryDTO);
return restUtils.processDistribution(item, summaryDTO, false);

}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

@Component
public class DistributionRunStatusUpdateProcessor {
Expand Down Expand Up @@ -69,21 +70,29 @@ private Map<String, ServiceException> updateBackStudentRecords(List<StudentCrede
String activityCode = getActivitCode(jobType);
final int totalCount = cList.size();
final int[] processedCount = {0};
List<UUID> studentIDs = cList.stream().map(StudentCredentialDistribution::getStudentID).toList();
cList.forEach(scd-> {
try {
final String token = restUtils.getAccessToken();
final String accessToken = restUtils.getAccessToken();
restUtils.updateStudentCredentialRecord(scd.getStudentID(),scd.getCredentialTypeCode(),scd.getPaperType(),
"NONGRADYERUN".equalsIgnoreCase(activityCode)? "IP" : scd.getDocumentStatusCode(),activityCode,token);
"NONGRADYERUN".equalsIgnoreCase(activityCode)? "IP" : scd.getDocumentStatusCode(),activityCode,accessToken);
LOGGER.debug("Dist Job [{}] / [{}] - update {} of {} student credential record: studentID, credentials, document status [{}, {}, {}]", batchId, activityCode, processedCount[0] + 1, totalCount, scd.getStudentID(), scd.getCredentialTypeCode(), scd.getDocumentStatusCode());
if(!StringUtils.equalsAnyIgnoreCase(jobType, "DISTRUN_YE", "NONGRADRUN", "REGALG", "TVRRUN")) {
restUtils.updateStudentGradRecord(scd.getStudentID(), batchId, activityCode, token);
LOGGER.debug("Dist Job [{}] / [{}] - update {} of {} student grad record: studentID, credentials, document status [{}, {}, {}]", batchId, activityCode, processedCount[0] + 1, totalCount, scd.getStudentID(), scd.getCredentialTypeCode(), scd.getDocumentStatusCode());
}
processedCount[0]++;
} catch (Exception e) {
unprocessedStudents.put(scd.getStudentID().toString(), new ServiceException(e));
}
});
studentIDs.forEach(uuid-> {
try {
if(!StringUtils.equalsAnyIgnoreCase(jobType, "REGALG", "TVRRUN")) {
restUtils.updateStudentGradRecord(uuid, batchId, activityCode);
LOGGER.debug("Dist Job [{}] / [{}] - update {} of {} student grad record: studentID [{}]", batchId, activityCode, processedCount[0] + 1, totalCount, uuid);
}
processedCount[0]++;
} catch (Exception e) {
unprocessedStudents.put(uuid.toString(), new ServiceException(e));
}
});
return unprocessedStudents;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ public class DistributionRunYearlyProcessor implements ItemProcessor<StudentCred
public StudentCredentialDistribution process(StudentCredentialDistribution item) throws Exception {
LOGGER.info("Processing partitionData = {}", item.getCredentialTypeCode());
summaryDTO.setBatchId(batchId);
return restUtils.processDistribution(item, summaryDTO);
return restUtils.processDistribution(item, summaryDTO, true);

}
}
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import org.springframework.beans.factory.annotation.Value;

import java.util.List;
import java.util.stream.Collectors;

public class EDWSnapshotSchoolProcessor implements ItemProcessor<String, List<Pair<String, List<SnapshotResponse>>>> {

Expand All @@ -31,7 +30,7 @@ public class EDWSnapshotSchoolProcessor implements ItemProcessor<String, List<Pa
public List<Pair<String, List<SnapshotResponse>>> process(String mincode) throws Exception {
summaryDTO.setBatchId(batchId);
LOGGER.debug("Processing partitionData for School: {} ", mincode);
List<SnapshotResponse> edwStudents = restUtils.getEDWSnapshotStudents(summaryDTO.getGradYear(), mincode, summaryDTO.getAccessToken());
List<SnapshotResponse> edwStudents = restUtils.getEDWSnapshotStudents(summaryDTO.getGradYear(), mincode);
if ("L".equals(summaryDTO.getOption())) {
List<SnapshotResponse> list = edwStudents.stream().filter(s -> StringUtils.isNotBlank(s.getGraduatedDate()) ||
("12".equals(s.getStudentGrade()) || "AD".equals(s.getStudentGrade()))).toList();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,15 @@ public class RunCertificateRegenerationProcessor implements ItemProcessor<UUID,

@Override
public Integer process(UUID key) throws Exception {
GraduationStudentRecordDistribution stuRec = restUtils.getStudentData(key.toString(), summaryDTO.getAccessToken());
GraduationStudentRecordDistribution stuRec = restUtils.getStudentData(key.toString());
if (stuRec != null) {
LOGGER.info("Processing partitionData: studentID = {}", stuRec.getStudentID());
summaryDTO.setBatchId(batchId);
try {
summaryDTO.setProcessedCount(summaryDTO.getProcessedCount() + 1L);
Integer count = restUtils.runRegenerateStudentCertificate(stuRec.getPen(), summaryDTO.getAccessToken());
Integer count = restUtils.runRegenerateStudentCertificate(stuRec.getPen());
if (count > 0) {
restUtils.updateStudentGradRecord(key, batchId, "CERTREGEN", summaryDTO.getAccessToken());
restUtils.updateStudentGradRecord(key, batchId, "CERTREGEN");
}
return count;
} catch(Exception e) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -227,24 +227,29 @@ void filterByStudentSearchRequest(List<StudentCredentialDistribution> eligibleSt
}
}
eligibleStudentSchoolDistricts.removeIf(scr->StringUtils.isNotBlank(scr.getSchoolOfRecord()) && !useFilterSchoolDistricts.contains(scr.getSchoolOfRecord()));
LOGGER.debug("Student Credential Distribution filtered by schoolCategory code {}: ", searchRequest.getSchoolCategoryCodes());
}
if(searchRequest != null && searchRequest.getDistricts() != null && !searchRequest.getDistricts().isEmpty()) {
eligibleStudentSchoolDistricts.removeIf(scr->StringUtils.isNotBlank(scr.getSchoolOfRecord()) && !searchRequest.getDistricts().contains(StringUtils.substring(scr.getSchoolOfRecord(), 0, 3)));
LOGGER.debug("Student Credential Distribution filtered by district code {}: ", searchRequest.getDistricts());
}
if(searchRequest != null && searchRequest.getSchoolOfRecords() != null && !searchRequest.getSchoolOfRecords().isEmpty()) {
eligibleStudentSchoolDistricts.removeIf(scr->StringUtils.isNotBlank(scr.getSchoolOfRecord()) && !searchRequest.getSchoolOfRecords().contains(scr.getSchoolOfRecord()));
LOGGER.debug("Student Credential Distribution filtered by schoolOfRecord code {}: ", searchRequest.getSchoolOfRecords());
}
if(searchRequest != null && searchRequest.getStudentIDs() != null && !searchRequest.getStudentIDs().isEmpty()) {
eligibleStudentSchoolDistricts.removeIf(scr->scr.getStudentID() != null && !searchRequest.getStudentIDs().contains(scr.getStudentID()));
LOGGER.debug("Student Credential Distribution filtered by student ID {}: ", searchRequest.getStudentIDs());
}
if(searchRequest != null && searchRequest.getPens() != null && !searchRequest.getPens().isEmpty()) {
eligibleStudentSchoolDistricts.removeIf(scr->StringUtils.isNotBlank(scr.getPen()) && !searchRequest.getPens().contains(scr.getPen()));
LOGGER.debug("Student Credential Distribution filtered by pen {}: ", searchRequest.getPens());
}
}

void filterOutDeceasedStudents(List<StudentCredentialDistribution> credentialList) {
LOGGER.debug("Total size of credential list: {}", credentialList.size());
List<UUID> deceasedIDs = restUtils.getDeceasedStudentIDs(credentialList.stream().map(StudentCredentialDistribution::getStudentID).distinct().toList(), restUtils.getAccessToken());
List<UUID> deceasedIDs = restUtils.getDeceasedStudentIDs(credentialList.stream().map(StudentCredentialDistribution::getStudentID).distinct().toList());
if (!deceasedIDs.isEmpty()) {
LOGGER.debug("Deceased students: {}", deceasedIDs.size());
credentialList.removeIf(cr -> deceasedIDs.contains(cr.getStudentID()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,14 +35,14 @@ public Map<String, ExecutionContext> partition(int gridSize) {
// Clean up existing reports before running new one
logger.debug("Delete School Reports for Monthly Distribution");
long startTime = System.currentTimeMillis();
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL", restUtils.getAccessToken());
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_SCHL");
long endTime = System.currentTimeMillis();
long diff = (endTime - startTime)/1000;
logger.debug("Old School Reports deleted in {} sec", diff);

startTime = System.currentTimeMillis();
logger.debug("Retrieve students for Monthly Distribution");
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData(restUtils.getAccessToken());
Mono<DistributionDataParallelDTO> parallelDTOMono = parallelDataFetch.fetchDistributionRequiredData();
DistributionDataParallelDTO parallelDTO = parallelDTOMono.block();
List<StudentCredentialDistribution> credentialList = new ArrayList<>();
if(parallelDTO != null) {
Expand All @@ -62,7 +62,6 @@ public Map<String, ExecutionContext> partition(int gridSize) {
return new HashMap<>();
}


@Override
protected JobExecution getJobExecution() {
return context;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public Map<String, ExecutionContext> partition(int gridSize) {
String transmissionType = jobParameters.getString("transmissionType");
PsiCredentialRequest req = (PsiCredentialRequest)jsonTransformer.unmarshall(searchRequest, PsiCredentialRequest.class);
String accessToken = restUtils.getAccessToken();
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_PSI", accessToken);
restUtils.deleteSchoolReportRecord("", "ADDRESS_LABEL_PSI");

List<PsiCredentialDistribution> credentialList = getRecordsForPSIUserReqDisRun(req,transmissionType,accessToken);
if(!credentialList.isEmpty()) {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package ca.bc.gov.educ.api.batchgraduation.reader;

import ca.bc.gov.educ.api.batchgraduation.model.ResponseObj;
import ca.bc.gov.educ.api.batchgraduation.model.StudentCredentialDistribution;
import ca.bc.gov.educ.api.batchgraduation.model.StudentSearchRequest;
import org.slf4j.Logger;
Expand All @@ -23,15 +22,10 @@ public class DistributionRunPartitionerUserReq extends BasePartitioner {

@Override
public Map<String, ExecutionContext> partition(int gridSize) {
ResponseObj res = restUtils.getTokenResponseObject();
String accessToken = null;
if (res != null) {
accessToken = res.getAccess_token();
}
JobParameters jobParameters = context.getJobParameters();
String credentialType = jobParameters.getString("credentialType");
StudentSearchRequest req = getStudentSearchRequest();
List<StudentCredentialDistribution> credentialList = restUtils.getStudentsForUserReqDisRun(credentialType,req,accessToken);
List<StudentCredentialDistribution> credentialList = restUtils.getStudentsForUserReqDisRun(credentialType,req);
if(!credentialList.isEmpty()) {
Map<String, ExecutionContext> map = getStringExecutionContextMap(gridSize, credentialList, credentialType);
LOGGER.info("Found {} in total running on {} partitions",credentialList.size(),map.size());
Expand Down
Loading

0 comments on commit a679055

Please sign in to comment.