allow to continue query when loading data

src/org/leolo/rail/nrd/AssoicationProcessor.java

@@ -35,7 +35,7 @@ public class AssoicationProcessor implements Runnable {
 		try(
 				BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));
 				Connection conn = DatabaseManager.getInstance().getConnection();
-				PreparedStatement pstmt = conn.prepareStatement("INSERT INTO train_assoc VALUES(null,?,?,?,?,?,?,?,?,?,?,?,?)")
+				PreparedStatement pstmt = conn.prepareStatement("INSERT INTO n_train_assoc VALUES(null,?,?,?,?,?,?,?,?,?,?,?,?)")
 		){
 			while(true) {
 				String line = br.readLine();

src/org/leolo/rail/nrd/DatabaseManager.java

@@ -79,6 +79,7 @@ public class DatabaseManager {
 		return conn;
 	}
 	
+	@Deprecated
 	public void clear() {
 		try(
 				Connection conn = getConnection();

src/org/leolo/rail/nrd/FileLoader.java

@@ -13,6 +13,9 @@ import java.io.OutputStreamWriter;
 import java.io.PrintWriter;
 import java.net.URL;
 import java.net.URLConnection;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.Base64;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -68,7 +71,8 @@ public class FileLoader {
 		if(!tempDir.exists()) {
 			tempDir.mkdirs();
 		}
-		DatabaseManager.getInstance().clear();
+//		DatabaseManager.getInstance().clear();
+		prepDB();
 		int countA=0, countT=0, countS=0;
 		int batchA=0, batchT=0, batchS=0;
 		try(
@@ -173,6 +177,7 @@ public class FileLoader {
 			log.error("Some job cannot be finished!");
 			System.exit(50);
 		}
+		finalizeDB();
 		DatabaseManager.getInstance().shutdown();
 		if(!tempDir.delete()) {
 			log.warn("Unable to remove temp dir!");
@@ -180,6 +185,64 @@ public class FileLoader {
 		log.info("Job finished!");
 	}
 	
+	private static final String [] TABLES = {
+			"tiploc","train_assoc","train_schedule",
+			"train_schedule_detail","train_schedule_location","train_error",
+	};
+	
+	private void finalizeDB() {
+		try(
+				Connection conn = DatabaseManager.getInstance().getConnection();
+				Statement stmt = conn.createStatement();
+		){
+			for(String table:TABLES) {
+				log.info("Handling {}", table);
+				stmt.execute("DROP TABLE IF EXISTS o_"+table);
+				stmt.execute("RENAME TABLE "+table+" TO o_"+table+",n_"+table+" TO "+table);
+			}
+			log.info("Creating extra index. This may take a while");
+			stmt.execute("ALTER TABLE train_schedule_location ADD INDEX (tiploc) USING BTREE");
+			log.info("Removing backup tables");
+			for(String table:TABLES) {
+				stmt.execute("DROP TABLE IF EXISTS o_"+table);
+			}
+		} catch (SQLException e) {
+			log.fatal(e.getMessage(), e);
+			System.exit(1);
+		}
+	}
+
+	private void prepDB() {
+		log.info("Preparing the database.");
+		StringBuilder sqls = new StringBuilder();
+		try(BufferedReader br = new BufferedReader(new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("org/leolo/rail/nrd/db.sql")))){
+			while(true) {
+				String line  = br.readLine();
+				if(line==null) {
+					break;
+				}
+				sqls.append(line);
+			}
+		} catch (IOException e) {
+			log.fatal(e.getMessage(), e);
+			System.exit(1);
+		}
+		log.info("Done loading SQLs from resource");
+		try(
+				Connection conn = DatabaseManager.getInstance().getConnection();
+				Statement stmt = conn.createStatement();
+		){
+			for(String sql:sqls.toString().split(";")) {
+				log.debug("SQL: {}", sql);
+				stmt.execute(sql);
+			}
+		} catch (SQLException e) {
+			log.fatal(e.getMessage(), e);
+			System.exit(1);
+		}
+		log.info("Done prepare the DB");
+	}
+	
 	private void captureFile() {
 		try {
 			URL url = new URL(ConfigurationManager.getInstance().get("remote.path").toString());

src/org/leolo/rail/nrd/ScheduleProcessor.java

@@ -33,7 +33,7 @@ public class ScheduleProcessor implements Runnable {
 	private Logger log = LogManager.getLogger(getClass());
 	private static TUIDDateFormat tdf = new TUIDDateFormat();
 
-	public static final String SEQ_ID = "0123456789ABCDEFGHJKLMNPRSTUVWXY";
+	public static final String SEQ_ID = "0123456789";
 	private static Random r = new Random();
 	private static class TUIDDateFormat extends DateFormat{
 		
@@ -129,10 +129,10 @@ public class ScheduleProcessor implements Runnable {
 		try(
 				BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));
 				Connection conn = DatabaseManager.getInstance().getConnection();
-				PreparedStatement sMain = conn.prepareStatement("INSERT INTO train_schedule VALUES (?,?,?,?,?,?,?)");
+				PreparedStatement sMain = conn.prepareStatement("INSERT INTO n_train_schedule VALUES (?,?,?,?,?,?,?)");
-				PreparedStatement sDetail = conn.prepareStatement("INSERT INTO train_schedule_detail VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
+				PreparedStatement sDetail = conn.prepareStatement("INSERT INTO n_train_schedule_detail VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
-				PreparedStatement sError = conn.prepareStatement("INSERT INTO train_error VALUES (?,?)");
+				PreparedStatement sError = conn.prepareStatement("INSERT INTO n_train_error VALUES (?,?)");
-				PreparedStatement sLoca = conn.prepareStatement("INSERT INTO train_schedule_location VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
+				PreparedStatement sLoca = conn.prepareStatement("INSERT INTO n_train_schedule_location VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
 		){
 			while(true) {
 				String line = br.readLine();

src/org/leolo/rail/nrd/TiplocProcessor.java

@@ -35,7 +35,7 @@ public class TiplocProcessor implements Runnable {
 		try(
 				BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));
 				Connection conn = DatabaseManager.getInstance().getConnection();
-				PreparedStatement pstmt = conn.prepareStatement("INSERT INTO tiploc VALUES (?,?,?,?,?,?)")
+				PreparedStatement pstmt = conn.prepareStatement("INSERT INTO n_tiploc VALUES (?,?,?,?,?,?)")
 		){
 			while(true) {
 				String line = br.readLine();

src/org/leolo/rail/nrd/db.sql

@@ -0,0 +1,88 @@
+DROP TABLE IF EXISTS `n_tiploc`;
+CREATE TABLE IF NOT EXISTS `n_tiploc` (
+  `tiploc_code` varchar(8) NOT NULL,
+  `nalco` varchar(6) DEFAULT NULL,
+  `stanox` varchar(5) DEFAULT NULL,
+  `crs` varchar(3) DEFAULT NULL,
+  `description` varchar(100) DEFAULT NULL,
+  `tps_desc` varchar(100) DEFAULT NULL,
+  PRIMARY KEY (`tiploc_code`),
+  KEY `nalco` (`nalco`),
+  KEY `stanox` (`stanox`),
+  KEY `crs` (`crs`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+DROP TABLE IF EXISTS `n_train_assoc`;
+CREATE TABLE IF NOT EXISTS `n_train_assoc` (
+  `assoc_id` int(11) NOT NULL AUTO_INCREMENT,
+  `main_train_uid` char(8) NOT NULL,
+  `assoc_train_uid` char(8) NOT NULL,
+  `start_date` date NOT NULL,
+  `end_date` date NOT NULL,
+  `days` char(7) NOT NULL DEFAULT '111111',
+  `assoc_cat` char(2) NOT NULL DEFAULT '',
+  `date_ind` char(1) NOT NULL DEFAULT '',
+  `tiploc` char(8) NOT NULL DEFAULT '',
+  `base_suffix` char(7) DEFAULT '',
+  `assoc_suffix` char(7) DEFAULT '',
+  `diagram_type` char(1) NOT NULL DEFAULT '',
+  `stp_ind` char(1) NOT NULL DEFAULT '',
+  PRIMARY KEY (`assoc_id`),
+  KEY `main_train_uid` (`main_train_uid`),
+  KEY `assoc_train_uid` (`assoc_train_uid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+DROP TABLE IF EXISTS `n_train_error`;
+CREATE TABLE IF NOT EXISTS `n_train_error` (
+  `train_uid` char(8) NOT NULL,
+  `segment_data` text NOT NULL,
+  PRIMARY KEY (`train_uid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+DROP TABLE IF EXISTS `n_train_schedule`;
+CREATE TABLE IF NOT EXISTS `n_train_schedule` (
+  `uid` char(12) NOT NULL,
+  `train_uid` char(6) NOT NULL,
+  `start_date` date NOT NULL,
+  `end_date` date NOT NULL,
+  `days` char(7) NOT NULL DEFAULT '1111111',
+  `status` char(10) NOT NULL,
+  `atoc` char(2) NOT NULL,
+  PRIMARY KEY (`uid`) USING BTREE,
+  KEY `train_uid` (`train_uid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+DROP TABLE IF EXISTS `n_train_schedule_detail`;
+CREATE TABLE IF NOT EXISTS `n_train_schedule_detail` (
+  `uid` char(12) NOT NULL,
+  `category` char(2) DEFAULT NULL,
+  `signal_id` char(4) DEFAULT NULL,
+  `headcode` char(4) DEFAULT NULL,
+  `course_ind` char(1) DEFAULT NULL,
+  `service_code` char(8) DEFAULT NULL,
+  `bus_sector` char(2) DEFAULT NULL,
+  `power_type` char(3) DEFAULT NULL,
+  `timing_load` char(4) DEFAULT NULL,
+  `speed` mediumint(8) unsigned DEFAULT NULL,
+  `op_chars` char(6) DEFAULT NULL,
+  `sleeper` char(1) DEFAULT NULL,
+  `resv` char(1) DEFAULT NULL,
+  `conn_ind` char(1) DEFAULT NULL,
+  `catering` char(4) DEFAULT NULL,
+  `branding` char(4) DEFAULT NULL,
+  PRIMARY KEY (`uid`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+DROP TABLE IF EXISTS `n_train_schedule_location`;
+CREATE TABLE IF NOT EXISTS `n_train_schedule_location` (
+  `tuid` char(12) NOT NULL,
+  `seq` int(10) unsigned NOT NULL,
+  `tiploc` char(8) DEFAULT NULL,
+  `arrival` time DEFAULT NULL,
+  `pub_arrival` time DEFAULT NULL,
+  `departure` time DEFAULT NULL,
+  `pub_departure` time DEFAULT NULL,
+  `pass` time DEFAULT NULL,
+  `platform` char(3) DEFAULT NULL,
+  `line` char(3) DEFAULT NULL,
+  `path` char(3) DEFAULT NULL,
+  `eng_allowance` time DEFAULT NULL,
+  `path_allowance` time DEFAULT NULL,
+  `perf_allowance` time DEFAULT NULL,
+  PRIMARY KEY (`tuid`,`seq`)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;