diff --git a/db/create_tables.php b/db/create_tables.php index bed8e851..96d8d40c 100755 --- a/db/create_tables.php +++ b/db/create_tables.php @@ -107,7 +107,6 @@ query_ngafid_db($query); - $query = "CREATE TABLE `user` ( `id` INT(11) NOT NULL AUTO_INCREMENT, `email` VARCHAR(128) NOT NULL, @@ -608,12 +607,13 @@ query_ngafid_db($query); $query = "CREATE TABLE `airsync_fleet_info` ( - `fleet_id` int(11) NOT NULL, - `api_key` varchar(32) NOT NULL, - `api_secret` varchar(64) NOT NULL, + `fleet_id` int(11) NOT NULL, + `airsync_fleet_name` TEXT NOT NULL, + `api_key` varchar(32) NOT NULL, + `api_secret` varchar(64) NOT NULL, `last_upload_time` timestamp ON UPDATE CURRENT_TIMESTAMP, - `timeout` int(11) DEFAULT NULL, - `mutex` TINYINT DEFAULT 0, + `timeout` int(11) DEFAULT NULL, + `override` tinyint(4) DEFAULT NULL, PRIMARY KEY(`fleet_id`), FOREIGN KEY(`fleet_id`) REFERENCES `fleet`(`id`) @@ -641,40 +641,6 @@ query_ngafid_db($query); } -if ($create_airsync) { - $query = "CREATE TABLE `airsync_fleet_info` ( - `fleet_id` int(11) NOT NULL, - `api_key` varchar(32) NOT NULL, - `api_secret` varchar(64) NOT NULL, - `last_upload_time` timestamp ON UPDATE CURRENT_TIMESTAMP, - `timeout` int(11) DEFAULT NULL, - `mutex` TINYINT DEFAULT 0, - KEY `airsync_fleet_id_fk` (`fleet_id`), - CONSTRAINT `airsync_fleet_id_fk` FOREIGN KEY (`fleet_id`) REFERENCES `fleet` (`id`) - );"; - - query_ngafid_db($query); - - $query = "CREATE TABLE `airsync_imports` ( - `id` int(11) NOT NULL, - `time_received` timestamp NULL DEFAULT NULL, - `upload_id` int(11) NOT NULL, - `fleet_id` int(11) NOT NULL, - `flight_id` int(11) DEFAULT NULL, - `tail` varchar(512) NOT NULL, - PRIMARY KEY (`id`), - KEY `airsync_imports_uploads_null_fk` (`upload_id`), - KEY `airsync_imports_fleet_id_fk` (`fleet_id`), - KEY `airsync_imports_flights_null_fk` (`flight_id`), - CONSTRAINT `airsync_imports_fleet_id_fk` FOREIGN KEY (`fleet_id`) REFERENCES `fleet` (`id`), - CONSTRAINT `airsync_imports_flights_null_fk` FOREIGN KEY (`flight_id`) REFERENCES `flights` (`id`), - CONSTRAINT `airsync_imports_uploads_null_fk` FOREIGN KEY (`upload_id`) REFERENCES `uploads` (`id`) - );"; - - query_ngafid_db($query); -} - - if (!$update_turn_to_final) { $query = "CREATE TABLE `turn_to_final` ( `flight_id` INT(11) NOT NULL, diff --git a/src/main/java/org/ngafid/WebServer.java b/src/main/java/org/ngafid/WebServer.java index 876effc6..c8924391 100755 --- a/src/main/java/org/ngafid/WebServer.java +++ b/src/main/java/org/ngafid/WebServer.java @@ -91,11 +91,11 @@ public LocalDateTime read(final JsonReader jsonReader) throws IOException { } MUSTACHE_TEMPLATE_DIR = System.getenv("MUSTACHE_TEMPLATE_DIR"); - Runtime.getRuntime().addShutdownHook(new Thread(() -> { - String message = "NGAFID WebServer has shutdown at " + LocalDateTime.now().format(DateTimeFormatter.ofPattern("MM-dd-yyyy HH:mm:ss")); - LOG.info(message); - sendAdminEmails(message, "", EmailType.ADMIN_SHUTDOWN_NOTIFICATION); - })); + // Runtime.getRuntime().addShutdownHook(new Thread(() -> { + // String message = "NGAFID WebServer has shutdown at " + LocalDateTime.now().format(DateTimeFormatter.ofPattern("MM-dd-yyyy HH:mm:ss")); + // LOG.info(message); + // sendAdminEmails(message, "", EmailType.ADMIN_SHUTDOWN_NOTIFICATION); + // })); } /** diff --git a/src/main/java/org/ngafid/accounts/AirSyncAircraft.java b/src/main/java/org/ngafid/accounts/AirSyncAircraft.java index a2ed86f6..24636c02 100644 --- a/src/main/java/org/ngafid/accounts/AirSyncAircraft.java +++ b/src/main/java/org/ngafid/accounts/AirSyncAircraft.java @@ -3,6 +3,7 @@ import java.net.*; import java.sql.*; import javax.net.ssl.HttpsURLConnection; +import java.nio.charset.StandardCharsets; import org.ngafid.WebServer; import org.ngafid.accounts.AirSyncAuth.AccessToken; @@ -12,6 +13,7 @@ import com.google.gson.Gson; import com.google.gson.reflect.TypeToken; +import com.google.gson.reflect.*; import java.time.LocalDateTime; import java.util.ArrayList; @@ -127,13 +129,15 @@ public Optional getLastImportTime(Connection connection) throws S * * @throws an exception if there is a network or dbms issue */ - private List getImportsHTTPS(HttpsURLConnection netConnection, AirSyncAuth authentication) throws Exception { + private List getImportsHTTPS(HttpsURLConnection netConnection, AirSyncAuth authentication) throws IOException { netConnection.setRequestMethod("GET"); netConnection.setDoOutput(true); netConnection.setRequestProperty("Authorization", authentication.bearerString()); - InputStream is = netConnection.getInputStream(); - byte [] respRaw = is.readAllBytes(); + byte[] respRaw; + try (InputStream is = netConnection.getInputStream()) { + respRaw = is.readAllBytes(); + } String resp = new String(respRaw).replaceAll("aircraft_id", "aircraftId"); resp = resp.replaceAll("tail_number", "tailNumber"); @@ -152,6 +156,38 @@ private List getImportsHTTPS(HttpsURLConnection netConnection, Ai return page; } + static class AirSyncAircraftAccountInfo { + public String account_token, name; + + public AirSyncAircraftAccountInfo() {} + } + + public String getAirSyncFleetName() { + try { + AirSyncAuth authentication = fleet.getAuth(); + HttpsURLConnection netConnection = (HttpsURLConnection) new URL(AirSyncEndpoints.AIRSYNC_ROOT + "/aircraft/accounts").openConnection(); + netConnection.setRequestMethod("GET"); + netConnection.setDoOutput(true); + netConnection.setRequestProperty("Authorization", authentication.bearerString()); + + byte[] respRaw; + try (InputStream is = netConnection.getInputStream()) { + respRaw = is.readAllBytes(); + } + + List info = gson.fromJson(new String(respRaw), new TypeToken>(){}.getType()); + + if (info.size() != 1) { + LOG.severe("AirSync aircraft appears for multiple fleets. We do not support this functionality currently..."); + System.exit(1); + } + + return info.get(0).name; + } catch (IOException e) { + return null; + } + } + /** * Gets ALL imports for this Aircraft * @@ -160,7 +196,7 @@ private List getImportsHTTPS(HttpsURLConnection netConnection, Ai * * @return a {@link List} of AirSyncImports */ - public List getImports(Connection connection, AirSyncFleet fleet) { + public List getImports(Connection connection, AirSyncFleet fleet) throws IOException { AirSyncAuth authentication = fleet.getAuth(); List imports = new LinkedList<>(); @@ -168,15 +204,11 @@ public List getImports(Connection connection, AirSyncFleet fleet) int nPage = 0; while (continueIteration) { - try { - HttpsURLConnection netConnection = (HttpsURLConnection) getAircraftLogURL(nPage++).openConnection(); - List page = getImportsHTTPS(netConnection, fleet.getAuth()); - - continueIteration = page.size() == AirSyncEndpoints.PAGE_SIZE; - imports.addAll(page); - } catch (Exception e) { - AirSync.handleAirSyncAPIException(e, authentication); - } + HttpsURLConnection netConnection = (HttpsURLConnection) getAircraftLogURL(nPage++).openConnection(); + List page = getImportsHTTPS(netConnection, fleet.getAuth()); + + continueIteration = page.size() == AirSyncEndpoints.PAGE_SIZE; + imports.addAll(page); } return imports; @@ -191,7 +223,7 @@ public List getImports(Connection connection, AirSyncFleet fleet) * * @return a {@link List} of AirSyncImports */ - public List getImportsAfterDate(Connection connection, AirSyncFleet fleet, LocalDateTime lastImportTime) { + public List getImportsAfterDate(Connection connection, AirSyncFleet fleet, LocalDateTime lastImportTime) throws IOException { AirSyncAuth authentication = fleet.getAuth(); List imports = new LinkedList<>(); @@ -200,15 +232,11 @@ public List getImportsAfterDate(Connection connection, AirSyncFle int nPage = 0; while (continueIteration) { - try { - HttpsURLConnection netConnection = (HttpsURLConnection) getAircraftLogURL(nPage++, lastImportTime).openConnection(); - List page = getImportsHTTPS(netConnection, authentication); - - continueIteration = page.size() == AirSyncEndpoints.PAGE_SIZE; - imports.addAll(page); - } catch (Exception e) { - AirSync.handleAirSyncAPIException(e, authentication); - } + HttpsURLConnection netConnection = (HttpsURLConnection) getAircraftLogURL(nPage++, lastImportTime).openConnection(); + List page = getImportsHTTPS(netConnection, authentication); + + continueIteration = page.size() == AirSyncEndpoints.PAGE_SIZE; + imports.addAll(page); } return imports; diff --git a/src/main/java/org/ngafid/accounts/AirSyncAuth.java b/src/main/java/org/ngafid/accounts/AirSyncAuth.java index b4766433..102a0ab1 100644 --- a/src/main/java/org/ngafid/accounts/AirSyncAuth.java +++ b/src/main/java/org/ngafid/accounts/AirSyncAuth.java @@ -45,7 +45,8 @@ class AccessToken { public AirSyncAuth(String apiKey, String apiSecret) { byte [] srcWord = (apiKey + ":" + apiSecret).getBytes(); this.hash = Base64.getEncoder().encode(srcWord); - + System.out.println("API Key = " + apiKey); + System.out.println("API Secret = " + apiSecret); this.requestAuthorization(); } @@ -69,13 +70,16 @@ public void requestAuthorization() { connection.setDoOutput(true); connection.setRequestProperty("Authorization", "Basic " + new String(this.hash)); - InputStream is = connection.getInputStream(); - byte [] respRaw = is.readAllBytes(); + try (InputStream is = connection.getInputStream()) { + byte [] respRaw = is.readAllBytes(); + + is.close(); - String resp = new String(respRaw).replaceAll("access_token", "accessToken"); + String resp = new String(respRaw).replaceAll("access_token", "accessToken"); - this.accessToken = gson.fromJson(resp, AccessToken.class); - this.issueTime = LocalDateTime.now(); + this.accessToken = gson.fromJson(resp, AccessToken.class); + this.issueTime = LocalDateTime.now(); + } } catch (IOException ie) { ie.printStackTrace(); System.err.println("FATAL: Unable to get a token from AirSync! Exiting due to fatal error."); diff --git a/src/main/java/org/ngafid/accounts/AirSyncFleet.java b/src/main/java/org/ngafid/accounts/AirSyncFleet.java index 8d808054..ad7599ec 100644 --- a/src/main/java/org/ngafid/accounts/AirSyncFleet.java +++ b/src/main/java/org/ngafid/accounts/AirSyncFleet.java @@ -7,6 +7,9 @@ import java.time.LocalDateTime; import java.lang.reflect.Type; import java.util.ArrayList; +import java.util.Comparator; +import java.util.Collections; +import java.util.stream.Collectors; import java.util.LinkedList; import java.util.List; import java.util.Optional; @@ -20,6 +23,7 @@ import org.ngafid.flights.AirSync; import org.ngafid.flights.AirSyncEndpoints; import org.ngafid.flights.AirSyncImport; +import org.ngafid.flights.MalformedFlightFileException; import com.google.gson.Gson; import com.google.gson.reflect.*; @@ -29,33 +33,37 @@ */ public class AirSyncFleet extends Fleet { private AirSyncAuth authCreds; + private String airsyncFleetName; private List aircraft; private transient LocalDateTime lastQueryTime; - //timeout in minutes + // timeout in minutes private int timeout = -1; // 1 Day private static final int DEFAULT_TIMEOUT = 1440; - private static AirSyncFleet [] fleets = null; - + private static AirSyncFleet[] fleets = null; + private static final Logger LOG = Logger.getLogger(AirSyncFleet.class.getName()); private static final Gson gson = WebServer.gson; /** - * Default constructor - * - * @param id the fleet id - * @param name this fleet's name - * @param airSyncAuth the credentials for this fleet - * @param lastQueryTime the last time this fleet was synced with AirSync - * @param timeout how long the fleet is set to wait before checking for updates again + * Default constructor + * + * @param id the fleet id + * @param fleetName the name of this fleet in the NGAFID + * @param airsyncFleetName the name of the fleet on Airsyncs service + * @param airSyncAuth the credentials for this fleet + * @param lastQueryTime the last time this fleet was synced with AirSync + * @param timeout how long the fleet is set to wait before checking for updates again */ - public AirSyncFleet(int id, String name, AirSyncAuth airSyncAuth, LocalDateTime lastQueryTime, int timeout) { - super(id, name); + public AirSyncFleet(int id, String fleetName, String airsyncFleetName, AirSyncAuth airSyncAuth, + LocalDateTime lastQueryTime, int timeout) { + super(id, fleetName); this.authCreds = airSyncAuth; + this.airsyncFleetName = airsyncFleetName; this.lastQueryTime = lastQueryTime; if (timeout <= 0) { @@ -72,16 +80,17 @@ public AirSyncFleet(int id, String name, AirSyncAuth airSyncAuth, LocalDateTime */ private AirSyncFleet(ResultSet resultSet) throws SQLException { super(resultSet.getInt(1), resultSet.getString(2)); - this.authCreds = new AirSyncAuth(resultSet.getString(3), resultSet.getString(4)); + this.airsyncFleetName = resultSet.getString(3); + this.authCreds = new AirSyncAuth(resultSet.getString(4), resultSet.getString(5)); - Timestamp timestamp = resultSet.getTimestamp(5); + Timestamp timestamp = resultSet.getTimestamp(6); if (timestamp == null) { this.lastQueryTime = LocalDateTime.MIN; } else { this.lastQueryTime = timestamp.toLocalDateTime(); } - int timeout = resultSet.getInt(6); + int timeout = resultSet.getInt(7); if (timeout <= 0) { this.timeout = DEFAULT_TIMEOUT; } else { @@ -91,9 +100,9 @@ private AirSyncFleet(ResultSet resultSet) throws SQLException { public boolean getOverride(Connection connection) throws SQLException { String query = """ - SELECT override from airsync_fleet_info WHERE - fleet_id = ? - """; + SELECT override from airsync_fleet_info WHERE + fleet_id = ? + """; try (PreparedStatement statement = connection.prepareStatement(query)) { statement.setInt(1, getId()); @@ -110,8 +119,8 @@ public boolean getOverride(Connection connection) throws SQLException { public void setOverride(Connection connection, boolean value) throws SQLException { String query = """ - UPDATE airsync_fleet_info SET override = ? WHERE fleet_id = ? - """; + UPDATE airsync_fleet_info SET override = ? WHERE fleet_id = ? + """; try (PreparedStatement statement = connection.prepareStatement(query)) { statement.setInt(1, value ? 1 : 0); @@ -121,58 +130,6 @@ public void setOverride(Connection connection, boolean value) throws SQLExceptio } } - public boolean compareAndSetMutex(Connection connection, int expected, int newValue) throws SQLException { - String queryText = """ - SELECT fleet_id, mutex FROM airsync_fleet_info WHERE - fleet_id = ? - AND mutex = ? FOR UPDATE - """; - - PreparedStatement query = connection.prepareStatement(queryText, ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_UPDATABLE); - query.setInt(1, getId()); - query.setInt(2, expected); - - ResultSet rs = query.executeQuery(); - if (rs.next()) { - rs.updateInt("mutex", newValue); - rs.updateRow(); - return true; - } else { - return false; - } - } - - /** - * Semaphore-style (P) mutex that allows for an entity (i.e. the daemon or user) to - * ask AirSync for updates - * - * In this case, the database holds the "signal" or lock flag - * - * @param connection the DBMS connection - * - * @return true if able to enter the critical section, false otherwise. This - * can allow for a busy-wait elsewhere - * - * @throws SQLException if the DMBS has an issue - */ - public boolean lock(Connection connection) throws SQLException { - return compareAndSetMutex(connection, 0, 1); - } - - /** - * Semaphore-style (V) mutex that allows for an entity (i.e. the daemon or user) to - * ask AirSync for updates - * - * In this case, the database holds the "signal" or lock flag - * - * @param connection the DBMS connection - * - * @throws SQLException if the DMBS has an issue - */ - public boolean unlock(Connection connection) throws SQLException { - return compareAndSetMutex(connection, 1, 0); - } - private LocalDateTime getLastQueryTime(Connection connection) throws SQLException { if (this.lastQueryTime == null) { String sql = "SELECT last_upload_time FROM airsync_fleet_info WHERE fleet_id = ?"; @@ -185,14 +142,14 @@ private LocalDateTime getLastQueryTime(Connection connection) throws SQLExceptio LocalDateTime lastQueryTime = null; if (resultSet.next()) { lastQueryTime = resultSet.getTimestamp(1).toLocalDateTime(); - } - + } + query.close(); return lastQueryTime; } return this.lastQueryTime; - } + } /** * Gets the timeout of the AirSyncFleet @@ -230,7 +187,7 @@ public int getTimeout(Connection connection) throws SQLException { * Gets the timeout this fleet chose in a human-readable form * * @param connection the DBMS connection - * @param fleetId the Fleets's id + * @param fleetId the Fleets's id * * @return the timeout as "X hours" or "XX minutes" * @@ -247,7 +204,7 @@ public static String getTimeout(Connection connection, int fleetId) throws SQLEx if (resultSet.next()) { timeout = resultSet.getInt(1); - } + } String formattedString = null; @@ -255,7 +212,7 @@ public static String getTimeout(Connection connection, int fleetId) throws SQLEx if (timeout >= 60) { formattedString = String.format("%d Hours", (timeout / 60)); } else { - formattedString = String.format("%d Minutes", timeout); + formattedString = String.format("%d Minutes", timeout); } } @@ -265,7 +222,7 @@ public static String getTimeout(Connection connection, int fleetId) throws SQLEx } /** - * Determines if the fleet is "out of data" i.e., is the last time we checked with airsync longer + * Determines if the fleet is "out of data" i.e., is the last time we checked with airsync longer * ago than the timeout specified? * * @param connection the DBMS connection @@ -275,21 +232,22 @@ public static String getTimeout(Connection connection, int fleetId) throws SQLEx public boolean isQueryOutdated(Connection connection) throws SQLException { LOG.info("dur " + Duration.between(getLastQueryTime(connection), LocalDateTime.now()).toMinutes()); LOG.info("dur " + getTimeout(connection)); - return (Duration.between(getLastQueryTime(connection), LocalDateTime.now()).toMinutes() >= getTimeout(connection)); + return (Duration.between(getLastQueryTime(connection), LocalDateTime.now()) + .toMinutes() >= getTimeout(connection)); } /** * This gets an existing AirSyncFleet by its fleet id * * @param connection is the DBMS connection - * @param fleetId is the id of the fleet to get + * @param fleetId is the id of the fleet to get * * @return an instance of an AirSyncFleet with the given id * * @throws SQLException if the DBMS has an error */ public static AirSyncFleet getAirSyncFleet(Connection connection, int fleetId) throws SQLException { - String sql = "SELECT fl.id, fl.fleet_name, sync.api_key, sync.api_secret, sync.last_upload_time, sync.timeout FROM fleet AS fl INNER JOIN airsync_fleet_info AS sync ON sync.fleet_id = fl.id WHERE fl.id = ?"; + String sql = "SELECT fl.id, fl.fleet_name, sync.airsync_fleet_name, sync.api_key, sync.api_secret, sync.last_upload_time, sync.timeout FROM fleet AS fl INNER JOIN airsync_fleet_info AS sync ON sync.fleet_id = fl.id WHERE fl.id = ?"; PreparedStatement query = connection.prepareStatement(sql); query.setInt(1, fleetId); @@ -311,15 +269,15 @@ public static AirSyncFleet getAirSyncFleet(Connection connection, int fleetId) t * Updates the fleets timeout for AirSync (how long to wait between checks) * * @param connection the DBMS connection - * @param user the User that is making the request + * @param user the User that is making the request * * @throws SQLException if the DBMS has an issue */ public void updateTimeout(Connection connection, User user, String newTimeout) throws SQLException { - // It is assumed that the UI will prevent non-admins from doing this, + // It is assumed that the UI will prevent non-admins from doing this, // but just to be safe we will have this check. if (user.isAdmin()) { - String [] timeoutTok = newTimeout.split("\\s"); + String[] timeoutTok = newTimeout.split("\\s"); int duration = Integer.parseInt(timeoutTok[0]); @@ -336,14 +294,15 @@ public void updateTimeout(Connection connection, User user, String newTimeout) t String sql = "UPDATE airsync_fleet_info SET timeout = ? WHERE fleet_id = ?"; PreparedStatement query = connection.prepareStatement(sql); - + query.setInt(1, timeoutMinutes); query.setInt(2, super.getId()); query.executeUpdate(); query.close(); } else { - LOG.severe("Non-admin user attempted to change AirSync settings! This should not happen! Offending user: " + user.getFullName()); + LOG.severe("Non-admin user attempted to change AirSync settings! This should not happen! Offending user: " + + user.getFullName()); } } @@ -361,7 +320,7 @@ public void setLastQueryTime(Connection connection) throws SQLException { /** * Sets the last query time for this fleet * - * @param time the time that the fleet was last queried + * @param time the time that the fleet was last queried * @param connection the DBMS connection * * @throws SQLException if there is a DBMS issue @@ -375,8 +334,8 @@ public void setLastQueryTime(LocalDateTime time, Connection connection) throws S query.executeUpdate(); query.close(); - //Force updating these variables the next time there - //is a check + // Force updating these variables the next time there + // is a check this.timeout = -1; this.lastQueryTime = null; } @@ -408,20 +367,27 @@ public List getAircraft() throws IOException { connection.setRequestMethod("GET"); connection.setDoOutput(true); - connection.setRequestProperty("Authorization", this.authCreds.bearerString()); + connection.setRequestProperty("Authorization", this.authCreds.bearerString()); + + byte[] respRaw; + try (InputStream is = connection.getInputStream()) { + respRaw = is.readAllBytes(); + } - InputStream is = connection.getInputStream(); - byte [] respRaw = is.readAllBytes(); - String resp = new String(respRaw).replaceAll("tail_number", "tailNumber"); - - Type target = new TypeToken>(){}.getType(); + + Type target = new TypeToken>() { + }.getType(); System.out.println(resp); - this.aircraft = gson.fromJson(resp, target); - for (AirSyncAircraft a : aircraft) a.initialize(this); + List aircraft = gson.fromJson(resp, target); + for (AirSyncAircraft a : aircraft) + a.initialize(this); + + this.aircraft = aircraft.stream().filter(a -> a.getAirSyncFleetName().equals(airsyncFleetName)) + .collect(Collectors.toList()); } - + return this.aircraft; } @@ -434,7 +400,7 @@ public List getAircraft() throws IOException { * * @throws SQLException if there is a DBMS issue */ - public static AirSyncFleet [] getAll(Connection connection) throws SQLException { + public static AirSyncFleet[] getAll(Connection connection) throws SQLException { String sql = "SELECT COUNT(*) FROM airsync_fleet_info"; PreparedStatement query = connection.prepareStatement(sql); @@ -451,7 +417,7 @@ public List getAircraft() throws IOException { query.close(); if (fleets == null || fleets.length != asFleetCount) { - sql = "SELECT fl.id, fl.fleet_name, sync.api_key, sync.api_secret, sync.last_upload_time, sync.timeout FROM fleet AS fl INNER JOIN airsync_fleet_info AS sync ON sync.fleet_id = fl.id"; + sql = "SELECT fl.id, fl.fleet_name, sync.airsync_fleet_name, sync.api_key, sync.api_secret, sync.last_upload_time, sync.timeout FROM fleet AS fl INNER JOIN airsync_fleet_info AS sync ON sync.fleet_id = fl.id"; query = connection.prepareStatement(sql); resultSet = query.executeQuery(); @@ -522,7 +488,6 @@ public String getLastUpdateTime(Connection connection) throws SQLException { /** * Updates this fleet with the AirSync servers - * This is not thread-safe and should be guarded with a mutex! * * @param connection the DBMS connection * @@ -530,39 +495,70 @@ public String getLastUpdateTime(Connection connection) throws SQLException { * * @throws SQLException if the DBMS has an issue */ - public String update(Connection connection) throws Exception { + public String update(Connection connection) throws SQLException { int nImports = 0; - List aircraft = this.getAircraft(); - for (AirSyncAircraft a : aircraft) { - List processedIds = getProcessedIds(connection); - - Optional aircraftLastImportTime = a.getLastImportTime(connection); + List aircraft = null; + try { + aircraft = this.getAircraft(); + } catch (IOException e) { + LOG.info("Failed to get list of aircraft"); + AirSync.crashGracefully(e); + } + for (AirSyncAircraft a : aircraft) { List imports; + List processedIds; + while (true) { + try { + processedIds = getProcessedIds(connection); + + Optional aircraftLastImportTime = a.getLastImportTime(connection); + + if (aircraftLastImportTime.isPresent()) { + // We must make the interval exclusive when asking the server for flights + LocalDateTime importTime = aircraftLastImportTime.get().plusSeconds(1); + imports = a.getImportsAfterDate(connection, this, importTime); + LOG.info(String.format("Getting imports for fleet %s after %s.", super.getName(), + importTime.toString())); + } else { + imports = a.getImports(connection, this); + LOG.info(String.format( + "Getting all imports for fleet %s, as there are no other uploads waiting for this fleet.", + super.getName())); + } - if (aircraftLastImportTime.isPresent()) { - // We must make the interval exclusive when asking the server for flights - LocalDateTime importTime = aircraftLastImportTime.get().plusSeconds(1); - imports = a.getImportsAfterDate(connection, this, importTime); - LOG.info(String.format("Getting imports for fleet %s after %s.", super.getName(), importTime.toString())); - } else { - imports = a.getImports(connection, this); - LOG.info(String.format("Getting all imports for fleet %s, as there are no other uploads waiting for this fleet.", super.getName())); + break; + } catch (IOException e) { + AirSync.handleAirSyncAPIException(e, authCreds); + } } - if (imports != null && !imports.isEmpty()) { - for (AirSyncImport i : imports) { - if (processedIds.contains(i.getId())) { - LOG.info("Skipping AirSync with upload id: " + i.getId() + " as it already exists in the database"); + Collections.sort(imports, new Comparator() { + public int compare(AirSyncImport left, AirSyncImport right) { + return left.getUploadTime().compareTo(right.getUploadTime()); + } + }); + + int i = 0; + while (i < imports.size()) { + try { + AirSyncImport im = imports.get(i); + + if (processedIds.contains(im.getId())) { + LOG.info("Skipping AirSync with upload id: " + im.getId() + + " as it already exists in the database"); } else { - i.process(connection); - System.out.println("Done processing " + i.getId()); + im.process(connection); + System.out.println("Done processing " + im.getId()); nImports++; } + + i++; + } catch (IOException e) { + AirSync.handleAirSyncAPIException(e, authCreds); + e.printStackTrace(); } - } else { - LOG.info("No imports found for aircraft: " + a.getTailNumber() + " in fleet " + super.getName() + ", continuing."); } } diff --git a/src/main/java/org/ngafid/flights/AirSync.java b/src/main/java/org/ngafid/flights/AirSync.java index 981c878d..4e78f2d9 100644 --- a/src/main/java/org/ngafid/flights/AirSync.java +++ b/src/main/java/org/ngafid/flights/AirSync.java @@ -23,9 +23,6 @@ * as many other methods may be used by the daemon */ public class AirSync { - //Used for debugging - static PrintStream logFile; - // How long the daemon will wait before making another request private static final long DEFAULT_WAIT_TIME = 10000; private static Connection connection = Database.getConnection(); @@ -35,9 +32,9 @@ public class AirSync { /** * Gracefully handles an exception from the AirSync API * - * @param e the exception caught - * @param authentication the authentication used at the time. We can use this - * to request a new one if its simply outdated. + * @param e the exception caught + * @param authentication the authentication used at the time. We can use this + * to request a new one if its simply outdated. */ public static void handleAirSyncAPIException(Exception e, AirSyncAuth authentication) { String message = e.getMessage(); @@ -45,9 +42,9 @@ public static void handleAirSyncAPIException(Exception e, AirSyncAuth authentica LOG.severe("Caught " + message + " when making AirSync request!"); if (message.contains("HTTP response code: 40")) { - LOG.severe("Bearer token is no longer valid (someone may have requested one elsewhere, or this daemon is running somewhere else!)."); + LOG.severe( + "Bearer token is no longer valid (someone may have requested one elsewhere, or this daemon is running somewhere else!)."); authentication.requestAuthorization(); - //logFile.println("Got exception at time " + LocalDateTime.now().toString() + ": " + e.getMessage()); } else if (message.contains("HTTP response code: 502")) { LOG.severe("Got a 502 error!"); crashGracefully(e); @@ -66,7 +63,8 @@ public static void sendAdminCrashNotification(String message) { ArrayList adminEmails = new ArrayList(Arrays.asList(NGAFID_ADMIN_EMAILS.split(";"))); ArrayList bccRecipients = new ArrayList(); - SendEmail.sendEmail(adminEmails, bccRecipients, "CRITICAL: AirSync Daemon Exception!", message, EmailType.AIRSYNC_DAEMON_CRASH); + SendEmail.sendEmail(adminEmails, bccRecipients, "CRITICAL: AirSync Daemon Exception!", message, + EmailType.AIRSYNC_DAEMON_CRASH); } /** @@ -78,8 +76,9 @@ public static void crashGracefully(Exception e) { System.err.println("FATAL: Exiting due to error " + e.getMessage() + "!"); e.printStackTrace(); - //TODO: format this as html! - StringBuilder sb = new StringBuilder("The NGAFID AirSync daemon has crashed at " + LocalDateTime.now().toString() + "!\n"); + // TODO: format this as html! + StringBuilder sb = new StringBuilder( + "The NGAFID AirSync daemon has crashed at " + LocalDateTime.now().toString() + "!\n"); sb.append("Exception caught: " + e.getMessage() + "\n"); sb.append("Stack trace:\n"); sb.append(ExceptionUtils.getStackTrace(e)); @@ -93,26 +92,24 @@ public static void crashGracefully(Exception e) { } /** - * This daemon's entry point. + * This daemon's entry point. * This is where the logic for how the daemon operates will be defined. * * @param args command line args */ - public static void main(String [] args) { + public static void main(String[] args) { LOG.info("AirSync daemon started"); try { LocalDateTime now = LocalDateTime.now(); - String timeStamp = new String() + now.getYear() + now.getMonthValue() + now.getDayOfMonth() + "-" + now.getHour() + now.getMinute() + now.getSecond(); - - //logFile = new PrintStream(new File("/var/log/ngafid/airsync_" + timeStamp + ".log")); - //logFile.println("Starting AirSync daemon error log at: " + now.toString()); + String timeStamp = new String() + now.getYear() + now.getMonthValue() + now.getDayOfMonth() + "-" + + now.getHour() + now.getMinute() + now.getSecond(); while (true) { - AirSyncFleet [] airSyncFleets = AirSyncFleet.getAll(connection); - + AirSyncFleet[] airSyncFleets = AirSyncFleet.getAll(connection); if (airSyncFleets == null || airSyncFleets.length == 0) { - LOG.severe("This instance of the NGAFID does not have any AirSync fleets configured. Please check the database and try again"); + LOG.severe( + "This instance of the NGAFID does not have any AirSync fleets configured. Please check the database and try again"); System.exit(1); } @@ -134,11 +131,6 @@ public static void main(String [] args) { LOG.info("Sleeping for " + waitTime / 1000 + "s."); Thread.sleep(waitTime); } - } catch (IOException e) { - String message = e.getMessage(); - LOG.info("Got exception: " + e.getMessage()); - if (message.contains("HTTP response code: 40")) - LOG.info("HINT: Your bearer token is either expired, or you are rate limited"); } catch (Exception e) { crashGracefully(e); } diff --git a/src/main/java/org/ngafid/flights/AirSyncImport.java b/src/main/java/org/ngafid/flights/AirSyncImport.java index 5ac52a00..4137925c 100644 --- a/src/main/java/org/ngafid/flights/AirSyncImport.java +++ b/src/main/java/org/ngafid/flights/AirSyncImport.java @@ -85,6 +85,10 @@ public void init(AirSyncFleet fleet, AirSyncAircraft aircraft) { this.localDateTimeEnd = LocalDateTime.parse(this.timeEnd.split("\\+")[0], DateTimeFormatter.ISO_LOCAL_DATE_TIME); } + public LocalDateTime getUploadTime() { + return localDateTimeUpload; + } + /** * Gets the uploader id of the AirSync user * @@ -94,7 +98,7 @@ public void init(AirSyncFleet fleet, AirSyncAircraft aircraft) { */ public static int getUploaderId() throws SQLException { if (AIRSYNC_UPLOADER_ID <= 0) { - String sql = "SELECT id FROM user WHERE first_name = 'airsync' AND last_name = 'user'"; + String sql = "SELECT id FROM user WHERE id = -1"; PreparedStatement query = Database.getConnection().prepareStatement(sql); ResultSet resultSet = query.executeQuery(); @@ -138,7 +142,7 @@ public static String getUploadIdentifier(int fleetId, int aircraftId, LocalDateT * * @throws MalformedFlightFileException if we get a bad file from AirSync */ - public void process(Connection connection) throws MalformedFlightFileException { + public void process(Connection connection) throws IOException { //Ensure there is data to read before proceeding... int count = 0; String identifier = getUploadIdentifier(this.fleet.getId(), aircraftId, this.localDateTimeStart); @@ -236,7 +240,7 @@ public void process(Connection connection) throws MalformedFlightFileException { this.createImport(connection, flight); CalculateExceedences.calculateExceedences(connection, uploadId, null); - } catch (IOException | FatalFlightFileException | FlightAlreadyExistsException e) { + } catch (FatalFlightFileException | FlightAlreadyExistsException e) { UploadException ue = new UploadException(e.getMessage(), e, csvName); try { FlightError.insertError(connection, uploadId, ue.getFilename(), ue.getMessage()); @@ -244,7 +248,7 @@ public void process(Connection connection) throws MalformedFlightFileException { } catch (SQLException se) { AirSync.crashGracefully(se); } - } catch (Exception e) { + } catch (SQLException e) { AirSync.crashGracefully(e); } } else { @@ -537,30 +541,25 @@ public static int getNumImports(Connection connection, int fleetId, String condi * * @return an InputStream instance with the import's CSV data */ - private InputStream getFileInputStream() { + private InputStream getFileInputStream() throws IOException { InputStream is = null; - try { - HttpsURLConnection connection = (HttpsURLConnection) new URL(String.format(AirSyncEndpoints.SINGLE_LOG, this.id)).openConnection(); + HttpsURLConnection connection = (HttpsURLConnection) new URL(String.format(AirSyncEndpoints.SINGLE_LOG, this.id)).openConnection(); - connection.setRequestMethod("GET"); - connection.setDoOutput(true); - connection.setRequestProperty("Authorization", this.fleet.getAuth().bearerString()); + connection.setRequestMethod("GET"); + connection.setDoOutput(true); + connection.setRequestProperty("Authorization", this.fleet.getAuth().bearerString()); - is = connection.getInputStream(); - byte [] respRaw = is.readAllBytes(); - - String resp = new String(respRaw).replaceAll("file_url", "fileUrl"); + is = connection.getInputStream(); + byte [] respRaw = is.readAllBytes(); + + String resp = new String(respRaw).replaceAll("file_url", "fileUrl"); - LogResponse log = WebServer.gson.fromJson(resp, LogResponse.class); - URL input = new URL(log.fileUrl); - LOG.info("Got URL for logfile " + log.fileUrl); + LogResponse log = WebServer.gson.fromJson(resp, LogResponse.class); + URL input = new URL(log.fileUrl); + LOG.info("Got URL for logfile " + log.fileUrl); - is = input.openStream(); - - } catch (IOException ie) { - AirSync.crashGracefully(ie); - } + is = input.openStream(); return is; } @@ -570,22 +569,15 @@ private InputStream getFileInputStream() { * * @param the size of the CSV buffer, in bytes */ - public int readCsvData() { - try { - InputStream is = getFileInputStream(); - + public int readCsvData() throws IOException { + try (InputStream is = getFileInputStream()) { if (is == null) { - AirSync.logFile.println("ERROR: Unable to read fileUrl from log endpoint for aircraft " + this.aircraftId + ": " + fileUrl + " with log: " + this.id + "."); return -1; } else { this.data = is.readAllBytes(); } - } catch (Exception e) { - AirSync.logFile.println("ERROR: Unable to read fileUrl for aircraftId " + this.aircraftId + ": " + fileUrl); - return -1; } - - // Return num of bytes read + return data.length; }