Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[WIP] Introduced new Flight File Processing System #82

Open
wants to merge 12 commits into
base: refactor-staging
Choose a base branch
from
Open

[WIP] Introduced new Flight File Processing System #82

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
82496f5
Initial implementation of new flight processing system
jkarns275 Apr 3, 2023
04ecd34
Part way though moving towards a flight builder
jkarns275 Apr 4, 2023
0ff0d4a
In progress commit for review
jkarns275 Apr 4, 2023
915df4a
Now compiles
jkarns275 Apr 4, 2023
1e4a5c5
In progress
jkarns275 Apr 6, 2023
195a6d4
Intermediate commit w/ interface for Aaron
jkarns275 Apr 7, 2023
ea1fe63
New Flight File Processing (#87)
AarC10 Apr 25, 2023
4b8ff21
Introduced flight file processing pipeline
jkarns275 Apr 27, 2023
917f401
Refactored airframe definitions; adding file info processing to the C…
jkarns275 Apr 27, 2023
3d44e18
Importing appears to work. Next step is validation
jkarns275 Apr 28, 2023
5d540ba
Batching database access
jkarns275 May 2, 2023
dff3a5b
Committed flights to db twice by mistake
jkarns275 May 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>11</source>
<target>11</target>
<source>16</source>
<target>16</target>
<compilerArgs>
<arg>-Xlint:all</arg>
<arg>-Xmaxwarns</arg>
Expand All @@ -145,8 +145,8 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>11</source>
<target>11</target>
<source>16</source>
<target>16</target>
</configuration>
</plugin>
</plugins>
Expand Down
64 changes: 33 additions & 31 deletions src/main/java/org/ngafid/CalculateExceedences.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,11 @@
import org.ngafid.filters.Conditional;
import org.ngafid.filters.Filter;
import org.ngafid.filters.Pair;
import java.util.logging.*;

public class CalculateExceedences {

private static final Logger LOG = Logger.getLogger(CalculateExceedences.class.getName());

static String timeSeriesName = "Lcl Time";
static String dateSeriesName = "Lcl Date";

Expand All @@ -55,75 +57,75 @@ public void processFlight(Connection connection, Flight flight, EventDefinition
int airframeNameId = flight.getAirframeNameId();
String flightFilename = flight.getFilename();

System.out.println("Processing flight: " + flightId + ", " + flightFilename);
LOG.info("Processing flight: " + flightId + ", " + flightFilename);

try {
System.out.println("Event is: '" + eventDefinition.getName() + "'");
LOG.info("Event is: '" + eventDefinition.getName() + "'");

//first check and see if this was actually a flight (RPM > 800)
Pair<Double,Double> minMaxRPM1 = DoubleTimeSeries.getMinMax(connection, flightId, "E1 RPM");
Pair<Double,Double> minMaxRPM2 = DoubleTimeSeries.getMinMax(connection, flightId, "E2 RPM");

System.out.println("minMaxRPM1: " + minMaxRPM1);
System.out.println("minMaxRPM2: " + minMaxRPM2);
// LOG.info("minMaxRPM1: " + minMaxRPM1);
// LOG.info("minMaxRPM2: " + minMaxRPM2);

if ((minMaxRPM1 == null && minMaxRPM2 == null) //both RPM values are null, can't calculate exceedence
|| (minMaxRPM2 == null && minMaxRPM1 != null && minMaxRPM1.second() < 800) //RPM2 is null, RPM1 is < 800
|| (minMaxRPM1 == null && minMaxRPM2 != null && minMaxRPM2.second() < 800) //RPM1 is null, RPM2 is < 800
|| (minMaxRPM1 != null && minMaxRPM1.second() < 800) && (minMaxRPM2 != null && minMaxRPM2.second() < 800)) { //RPM1 and RPM2 < 800
//couldn't calculate exceedences for this flight because the engines never kicked on (it didn't fly)
System.out.println("engines never turned on, setting flight_processed.had_error = 1");
LOG.info("engines never turned on, setting flight_processed.had_error = 1");

if (uploadProcessedEmail != null) uploadProcessedEmail.addExceedenceError(flightFilename, "could not calculate exceedences for flight " + flightId + ", '" + flightFilename + "' - engines never turned on");

PreparedStatement stmt = connection.prepareStatement("INSERT INTO flight_processed SET fleet_id = ?, flight_id = ?, event_definition_id = ?, count = 0, had_error = 1");
stmt.setInt(1, fleetId);
stmt.setInt(2, flightId);
stmt.setInt(3, eventDefinition.getId());
System.out.println(stmt.toString());
// LOG.info(stmt.toString());
stmt.executeUpdate();
stmt.close();
return;
}

TreeSet<String> columnNames = eventDefinition.getColumnNames();
System.out.println("Number of Column Name(s): [ " + columnNames.size() + " ]");
// LOG.info("Number of Column Name(s): [ " + columnNames.size() + " ]");

//first test and see if min/max values can violate exceedence, otherwise we can skip
conditional.reset();
for (String columnName : columnNames) {
Pair<Double,Double> minMax = DoubleTimeSeries.getMinMax(connection, flightId, columnName);

if (minMax == null) {
System.out.println("minMax was null, setting flight_processed.had_error = 1");
LOG.info("minMax was null, setting flight_processed.had_error = 1");
//couldn't calculate this exceedence because at least one of the columns was missing
if (uploadProcessedEmail != null) uploadProcessedEmail.addExceedenceError(flightFilename, "could not calculate '" + eventDefinition.getName() + "' for flight " + flightId + ", '" + flightFilename + "' - " + columnName + " was missing");

PreparedStatement stmt = connection.prepareStatement("INSERT INTO flight_processed SET fleet_id = ?, flight_id = ?, event_definition_id = ?, count = 0, had_error = 1");
stmt.setInt(1, fleetId);
stmt.setInt(2, flightId);
stmt.setInt(3, eventDefinition.getId());
System.out.println(stmt.toString());
// LOG.info(stmt.toString());
stmt.executeUpdate();
stmt.close();
return;
}

System.out.println(columnName + ", min: " + minMax.first() + ", max: " + minMax.second());
LOG.info(columnName + ", min: " + minMax.first() + ", max: " + minMax.second());
conditional.set(columnName, minMax);
}

System.out.println("Post-set conditional: " + conditional.toString());
LOG.info("Post-set conditional: " + conditional.toString());
boolean result = conditional.evaluate();
System.out.println("overall result: " + result);
LOG.info("overall result: " + result);

if (!result) {
//this flight could not have caused one of these events
PreparedStatement stmt = connection.prepareStatement("INSERT INTO flight_processed SET fleet_id = ?, flight_id = ?, event_definition_id = ?, count = 0, had_error = 0");
stmt.setInt(1, fleetId);
stmt.setInt(2, flightId);
stmt.setInt(3, eventDefinition.getId());
System.out.println(stmt.toString());
// LOG.info(stmt.toString());
stmt.executeUpdate();
stmt.close();

Expand All @@ -136,14 +138,14 @@ public void processFlight(Connection connection, Flight flight, EventDefinition

if (timeSeries == null || dateSeries == null) {
//couldn't calculate this exceedence because the date or time column was missing
System.out.println("time series or date series was missing, setting flight_processed.had_error = 1");
LOG.info("time series or date series was missing, setting flight_processed.had_error = 1");
if (uploadProcessedEmail != null) uploadProcessedEmail.addExceedenceError(flightFilename, "could not calculate exceedences for flight " + flightId + ", '" + flightFilename + "' - date or time was missing");

PreparedStatement stmt = connection.prepareStatement("INSERT INTO flight_processed SET fleet_id = ?, flight_id = ?, event_definition_id = ?, count = 0, had_error = 1");
stmt.setInt(1, fleetId);
stmt.setInt(2, flightId);
stmt.setInt(3, eventDefinition.getId());
System.out.println(stmt.toString());
LOG.info(stmt.toString());
stmt.executeUpdate();
stmt.close();
return;
Expand Down Expand Up @@ -174,23 +176,23 @@ public void processFlight(Connection connection, Flight flight, EventDefinition
lineNumber = i;
double currentValue = doubleSeries[0].get(i);

//System.out.println("Pre-set conditional: " + conditional.toString());
//LOG.info("Pre-set conditional: " + conditional.toString());

conditional.reset();
for (DoubleTimeSeries series : doubleSeries) {
conditional.set(series.getName(), series.get(i));
}
//System.out.println("Post-set conditional: " + conditional.toString());
//LOG.info("Post-set conditional: " + conditional.toString());

result = conditional.evaluate();

//System.out.println(conditional + ", result: " + result);
//LOG.info(conditional + ", result: " + result);

if (!result) {
if (startTime != null) {
//we're tracking an event, so increment the stopCount
stopCount++;
System.out.println("stopCount: " + stopCount + " with on line: " + lineNumber );
LOG.info("stopCount: " + stopCount + " with on line: " + lineNumber );

if (stopCount == stopBuffer) {
System.err.println("Stop count (" + stopCount + ") reached the stop buffer (" + stopBuffer + "), new event created!");
Expand Down Expand Up @@ -224,7 +226,7 @@ public void processFlight(Connection connection, Flight flight, EventDefinition
startLine = lineNumber;
severity = eventDefinition.getSeverity(doubleSeries, i);

System.out.println("start date time: " + startTime + ", start line number: " + startLine);
LOG.info("start date time: " + startTime + ", start line number: " + startLine);
}
endLine = lineNumber;
endTime = dateSeries.get(i) + " " + timeSeries.get(i);
Expand All @@ -240,11 +242,11 @@ public void processFlight(Connection connection, Flight flight, EventDefinition
Event event = new Event(startTime, endTime, startLine, endLine, severity);
eventList.add( event );
}
System.out.println("");
LOG.info("");

for (i = 0; i < eventList.size(); i++) {
Event event = eventList.get(i);
System.out.println( "Event : [line: " + event.getStartLine() + " to " + event.getEndLine() + ", time: " + event.getStartTime() + " to " + event.getEndTime() + "]" );
LOG.info( "Event : [line: " + event.getStartLine() + " to " + event.getEndLine() + ", time: " + event.getStartTime() + " to " + event.getEndTime() + "]" );
if (uploadProcessedEmail != null) uploadProcessedEmail.addExceedence(flightFilename, "flight " + flightId + ", '" + flightFilename + "' - '" + eventDefinition.getName() + "' from " + event.getStartTime() + " to " + event.getEndTime());
}

Expand Down Expand Up @@ -284,7 +286,7 @@ public void processFlight(Connection connection, Flight flight, EventDefinition
stmt.setDouble(8, sumSeverity);
stmt.setDouble(9, minSeverity);
stmt.setDouble(10, maxSeverity);
System.out.println(stmt.toString());
LOG.info(stmt.toString());
stmt.executeUpdate();
stmt.close();

Expand All @@ -295,7 +297,7 @@ public void processFlight(Connection connection, Flight flight, EventDefinition
stmt.setInt(1, fleetId);
stmt.setInt(2, flightId);
stmt.setInt(3, eventDefinition.getId());
System.out.println(stmt.toString());
LOG.info(stmt.toString());
stmt.executeUpdate();
stmt.close();

Expand All @@ -316,14 +318,14 @@ public static void calculateExceedences(Connection connection, int uploadId, Upl
if (allEvents == null) {
allEvents = EventDefinition.getAll(connection, "id > ?", new Object[]{0});
}
System.out.println("n events = " + allEvents.size());
LOG.info("n events = " + allEvents.size());

int airframeTypeId = Airframes.getTypeId(connection, "Fixed Wing");

for (int i = 0; i < allEvents.size(); i++) {
//process events for this event type
EventDefinition currentDefinition = allEvents.get(i);
System.out.println("\t" + currentDefinition.toString());
LOG.info("\t" + currentDefinition.toString());

CalculateExceedences currentCalculator = new CalculateExceedences(currentDefinition);

Expand All @@ -349,7 +351,7 @@ public static void calculateExceedences(Connection connection, int uploadId, Upl
Instant end = Instant.now();
long elapsed_millis = Duration.between(start, end).toMillis();
double elapsed_seconds = ((double) elapsed_millis) / 1000;
System.out.println("finished in " + elapsed_seconds);
LOG.info("finished in " + elapsed_seconds);

if (uploadProcessedEmail != null) uploadProcessedEmail.setExceedencesElapsedTime(elapsed_seconds);
}
Expand All @@ -365,11 +367,11 @@ public static void main(String[] arguments) {
connection = Database.resetConnection();
Instant start = Instant.now();
ArrayList<EventDefinition> allEvents = EventDefinition.getAll(connection, "id > ?", new Object[]{0});
System.out.println("n events = " + allEvents.size());
LOG.info("n events = " + allEvents.size());
for (int i = 0; i < allEvents.size(); i++) {
//process events for this event type
EventDefinition currentDefinition = allEvents.get(i);
System.out.println("\t" + currentDefinition.toString());
LOG.info("\t" + currentDefinition.toString());

CalculateExceedences currentCalculator = new CalculateExceedences(currentDefinition);

Expand All @@ -395,7 +397,7 @@ public static void main(String[] arguments) {
Instant end = Instant.now();
long elapsed_millis = Duration.between(start, end).toMillis();
double elapsed_seconds = ((double) elapsed_millis) / 1000;
System.out.println("finished in " + elapsed_seconds);
LOG.info("finished in " + elapsed_seconds);

try {
Thread.sleep(3000);
Expand Down
Loading