london-transport
/
nr-data
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity
Browse Source

allow to continue query when loading data

develop
LO Kam Tao Leo 4 years ago
parent
f49d900fdc
commit
c39193ec3d
6 changed files with 160 additions and 8 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      src/org/leolo/rail/nrd/AssoicationProcessor.java
  2. 1
      src/org/leolo/rail/nrd/DatabaseManager.java
  3. 65
      src/org/leolo/rail/nrd/FileLoader.java
  4. 10
      src/org/leolo/rail/nrd/ScheduleProcessor.java
  5. 2
      src/org/leolo/rail/nrd/TiplocProcessor.java
  6. 88
      src/org/leolo/rail/nrd/db.sql

2
src/org/leolo/rail/nrd/AssoicationProcessor.java
Unescape View File

@ -35,7 +35,7 @@ public class AssoicationProcessor implements Runnable {
try(
BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));
Connection conn = DatabaseManager.getInstance().getConnection();
PreparedStatement pstmt = conn.prepareStatement("INSERT INTO train_assoc VALUES(null,?,?,?,?,?,?,?,?,?,?,?,?)")
PreparedStatement pstmt = conn.prepareStatement("INSERT INTO n_train_assoc VALUES(null,?,?,?,?,?,?,?,?,?,?,?,?)")
){
while(true) {
String line = br.readLine();

1
src/org/leolo/rail/nrd/DatabaseManager.java
Unescape View File

@ -79,6 +79,7 @@ public class DatabaseManager {
return conn;
}
@Deprecated
public void clear() {
try(
Connection conn = getConnection();

65
src/org/leolo/rail/nrd/FileLoader.java
Unescape View File

@ -13,6 +13,9 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLConnection;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Base64;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
@ -68,7 +71,8 @@ public class FileLoader {
if(!tempDir.exists()) {
tempDir.mkdirs();
}
DatabaseManager.getInstance().clear();
// DatabaseManager.getInstance().clear();
prepDB();
int countA=0, countT=0, countS=0;
int batchA=0, batchT=0, batchS=0;
try(
@ -173,13 +177,72 @@ public class FileLoader {
log.error("Some job cannot be finished!");
System.exit(50);
}
finalizeDB();
DatabaseManager.getInstance().shutdown();
if(!tempDir.delete()) {
log.warn("Unable to remove temp dir!");
}
log.info("Job finished!");
}
private static final String [] TABLES = {
"tiploc","train_assoc","train_schedule",
"train_schedule_detail","train_schedule_location","train_error",
};
private void finalizeDB() {
try(
Connection conn = DatabaseManager.getInstance().getConnection();
Statement stmt = conn.createStatement();
){
for(String table:TABLES) {
log.info("Handling {}", table);
stmt.execute("DROP TABLE IF EXISTS o_"+table);
stmt.execute("RENAME TABLE "+table+" TO o_"+table+",n_"+table+" TO "+table);
}
log.info("Creating extra index. This may take a while");
stmt.execute("ALTER TABLE train_schedule_location ADD INDEX (tiploc) USING BTREE");
log.info("Removing backup tables");
for(String table:TABLES) {
stmt.execute("DROP TABLE IF EXISTS o_"+table);
}
} catch (SQLException e) {
log.fatal(e.getMessage(), e);
System.exit(1);
}
}
private void prepDB() {
log.info("Preparing the database.");
StringBuilder sqls = new StringBuilder();
try(BufferedReader br = new BufferedReader(new InputStreamReader(Thread.currentThread().getContextClassLoader().getResourceAsStream("org/leolo/rail/nrd/db.sql")))){
while(true) {
String line = br.readLine();
if(line==null) {
break;
}
sqls.append(line);
}
} catch (IOException e) {
log.fatal(e.getMessage(), e);
System.exit(1);
}
log.info("Done loading SQLs from resource");
try(
Connection conn = DatabaseManager.getInstance().getConnection();
Statement stmt = conn.createStatement();
){
for(String sql:sqls.toString().split(";")) {
log.debug("SQL: {}", sql);
stmt.execute(sql);
}
} catch (SQLException e) {
log.fatal(e.getMessage(), e);
System.exit(1);
}
log.info("Done prepare the DB");
}
private void captureFile() {
try {
URL url = new URL(ConfigurationManager.getInstance().get("remote.path").toString());

10
src/org/leolo/rail/nrd/ScheduleProcessor.java
Unescape View File

@ -33,7 +33,7 @@ public class ScheduleProcessor implements Runnable {
private Logger log = LogManager.getLogger(getClass());
private static TUIDDateFormat tdf = new TUIDDateFormat();
public static final String SEQ_ID = "0123456789ABCDEFGHJKLMNPRSTUVWXY";
public static final String SEQ_ID = "0123456789";
private static Random r = new Random();
private static class TUIDDateFormat extends DateFormat{
@ -129,10 +129,10 @@ public class ScheduleProcessor implements Runnable {
try(
BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));
Connection conn = DatabaseManager.getInstance().getConnection();
PreparedStatement sMain = conn.prepareStatement("INSERT INTO train_schedule VALUES (?,?,?,?,?,?,?)");
PreparedStatement sDetail = conn.prepareStatement("INSERT INTO train_schedule_detail VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
PreparedStatement sError = conn.prepareStatement("INSERT INTO train_error VALUES (?,?)");
PreparedStatement sLoca = conn.prepareStatement("INSERT INTO train_schedule_location VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
PreparedStatement sMain = conn.prepareStatement("INSERT INTO n_train_schedule VALUES (?,?,?,?,?,?,?)");
PreparedStatement sDetail = conn.prepareStatement("INSERT INTO n_train_schedule_detail VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
PreparedStatement sError = conn.prepareStatement("INSERT INTO n_train_error VALUES (?,?)");
PreparedStatement sLoca = conn.prepareStatement("INSERT INTO n_train_schedule_location VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
){
while(true) {
String line = br.readLine();

2
src/org/leolo/rail/nrd/TiplocProcessor.java
Unescape View File

@ -35,7 +35,7 @@ public class TiplocProcessor implements Runnable {
try(
BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName))));
Connection conn = DatabaseManager.getInstance().getConnection();
PreparedStatement pstmt = conn.prepareStatement("INSERT INTO tiploc VALUES (?,?,?,?,?,?)")
PreparedStatement pstmt = conn.prepareStatement("INSERT INTO n_tiploc VALUES (?,?,?,?,?,?)")
){
while(true) {
String line = br.readLine();

88
src/org/leolo/rail/nrd/db.sql
Unescape View File

@ -0,0 +1,88 @@
DROP TABLE IF EXISTS `n_tiploc`;
CREATE TABLE IF NOT EXISTS `n_tiploc` (
`tiploc_code` varchar(8) NOT NULL,
`nalco` varchar(6) DEFAULT NULL,
`stanox` varchar(5) DEFAULT NULL,
`crs` varchar(3) DEFAULT NULL,
`description` varchar(100) DEFAULT NULL,
`tps_desc` varchar(100) DEFAULT NULL,
PRIMARY KEY (`tiploc_code`),
KEY `nalco` (`nalco`),
KEY `stanox` (`stanox`),
KEY `crs` (`crs`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
DROP TABLE IF EXISTS `n_train_assoc`;
CREATE TABLE IF NOT EXISTS `n_train_assoc` (
`assoc_id` int(11) NOT NULL AUTO_INCREMENT,
`main_train_uid` char(8) NOT NULL,
`assoc_train_uid` char(8) NOT NULL,
`start_date` date NOT NULL,
`end_date` date NOT NULL,
`days` char(7) NOT NULL DEFAULT '111111',
`assoc_cat` char(2) NOT NULL DEFAULT '',
`date_ind` char(1) NOT NULL DEFAULT '',
`tiploc` char(8) NOT NULL DEFAULT '',
`base_suffix` char(7) DEFAULT '',
`assoc_suffix` char(7) DEFAULT '',
`diagram_type` char(1) NOT NULL DEFAULT '',
`stp_ind` char(1) NOT NULL DEFAULT '',
PRIMARY KEY (`assoc_id`),
KEY `main_train_uid` (`main_train_uid`),
KEY `assoc_train_uid` (`assoc_train_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
DROP TABLE IF EXISTS `n_train_error`;
CREATE TABLE IF NOT EXISTS `n_train_error` (
`train_uid` char(8) NOT NULL,
`segment_data` text NOT NULL,
PRIMARY KEY (`train_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
DROP TABLE IF EXISTS `n_train_schedule`;
CREATE TABLE IF NOT EXISTS `n_train_schedule` (
`uid` char(12) NOT NULL,
`train_uid` char(6) NOT NULL,
`start_date` date NOT NULL,
`end_date` date NOT NULL,
`days` char(7) NOT NULL DEFAULT '1111111',
`status` char(10) NOT NULL,
`atoc` char(2) NOT NULL,
PRIMARY KEY (`uid`) USING BTREE,
KEY `train_uid` (`train_uid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
DROP TABLE IF EXISTS `n_train_schedule_detail`;
CREATE TABLE IF NOT EXISTS `n_train_schedule_detail` (
`uid` char(12) NOT NULL,
`category` char(2) DEFAULT NULL,
`signal_id` char(4) DEFAULT NULL,
`headcode` char(4) DEFAULT NULL,
`course_ind` char(1) DEFAULT NULL,
`service_code` char(8) DEFAULT NULL,
`bus_sector` char(2) DEFAULT NULL,
`power_type` char(3) DEFAULT NULL,
`timing_load` char(4) DEFAULT NULL,
`speed` mediumint(8) unsigned DEFAULT NULL,
`op_chars` char(6) DEFAULT NULL,
`sleeper` char(1) DEFAULT NULL,
`resv` char(1) DEFAULT NULL,
`conn_ind` char(1) DEFAULT NULL,
`catering` char(4) DEFAULT NULL,
`branding` char(4) DEFAULT NULL,
PRIMARY KEY (`uid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
DROP TABLE IF EXISTS `n_train_schedule_location`;
CREATE TABLE IF NOT EXISTS `n_train_schedule_location` (
`tuid` char(12) NOT NULL,
`seq` int(10) unsigned NOT NULL,
`tiploc` char(8) DEFAULT NULL,
`arrival` time DEFAULT NULL,
`pub_arrival` time DEFAULT NULL,
`departure` time DEFAULT NULL,
`pub_departure` time DEFAULT NULL,
`pass` time DEFAULT NULL,
`platform` char(3) DEFAULT NULL,
`line` char(3) DEFAULT NULL,
`path` char(3) DEFAULT NULL,
`eng_allowance` time DEFAULT NULL,
`path_allowance` time DEFAULT NULL,
`perf_allowance` time DEFAULT NULL,
PRIMARY KEY (`tuid`,`seq`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
Write Preview
Loading…
Cancel
Save