diff --git a/pom.xml b/pom.xml
index db931e6..3ee36d2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -44,12 +44,12 @@
org.mariadb.jdbc
mariadb-java-client
- 2.7.2
+ 2.7.6
org.json
json
- 20210307
+ 20220924
org.quartz-scheduler
@@ -67,6 +67,11 @@
log4j-slf4j-impl
${ver-log4j2}
+
+ org.json
+ json
+ 20220924
+
@@ -119,8 +124,8 @@
org.apache.maven.plugins
maven-compiler-plugin
- 8
- 8
+ 11
+ 11
diff --git a/src/main/java/org/leolo/nrdatad/Constants.java b/src/main/java/org/leolo/nrdatad/Constants.java
index c955249..07a083c 100644
--- a/src/main/java/org/leolo/nrdatad/Constants.java
+++ b/src/main/java/org/leolo/nrdatad/Constants.java
@@ -12,5 +12,11 @@ public final class Constants {
public static class Configuration {
public static final String ALWAYS_RUN_LTP = "cron.ltp.always";
public static final String ALWAYS_RUN_REF_DATA = "cron.ref.always";
+ public static final String NETWORK_RAIL_USER = "network.user";
+ public static final String NETWORK_RAIL_PASSWORD = "network.pwd";
+ }
+
+ public static class NetworkRailURI{
+ public static final String CORPUS_URL = "https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS";
}
}
diff --git a/src/main/java/org/leolo/nrdatad/db/BaseDao.java b/src/main/java/org/leolo/nrdatad/db/BaseDao.java
index 383a8a9..e78fd13 100644
--- a/src/main/java/org/leolo/nrdatad/db/BaseDao.java
+++ b/src/main/java/org/leolo/nrdatad/db/BaseDao.java
@@ -20,7 +20,7 @@ public abstract class BaseDao {
}
protected void setString(PreparedStatement stmt, int pos, @Nullable String data) throws SQLException{
- if(data==null){
+ if(data==null||data.strip().length()==0){
stmt.setNull(pos, Types.CHAR);
}else{
stmt.setString(pos, data);
diff --git a/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java b/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java
index 8c125b9..77d87ed 100644
--- a/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java
+++ b/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java
@@ -24,4 +24,6 @@ public abstract class CORPUSDao extends BaseDao{
add(corpus);
}
}
+
+ public abstract void truncateTable() throws SQLException;
}
diff --git a/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java b/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java
index b738397..f9b99bf 100644
--- a/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java
+++ b/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java
@@ -13,4 +13,6 @@ public interface DatabaseManager {
public Connection getConnection() throws SQLException;
public MetadataDao getMetadataDao();
+
+ public CORPUSDao getCORPUSDao();
}
diff --git a/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java b/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java
index f1d13e1..4d9c847 100644
--- a/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java
+++ b/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java
@@ -7,6 +7,7 @@ import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
+import java.sql.Statement;
import java.util.Collection;
public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao {
@@ -36,6 +37,7 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao {
setString(pstmt, 6, corpus.getLongDescription());
setString(pstmt, 7, corpus.getShortDescription());
pstmt.executeUpdate();
+ conn.commit();
}
}
@@ -58,6 +60,17 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao {
pstmt.addBatch();
}
pstmt.executeBatch();
+ conn.commit();
+ }
+ }
+
+ @Override
+ public void truncateTable() throws SQLException {
+ try(
+ Connection conn = getConnection();
+ Statement stmt = conn.createStatement()
+ ){
+ stmt.executeQuery("TRUNCATE TABLE corpus");
}
}
}
diff --git a/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java b/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java
index d27e98f..0b88ba4 100644
--- a/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java
+++ b/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java
@@ -3,6 +3,7 @@ package org.leolo.nrdatad.db.mariadb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.leolo.nrdatad.ConfigurationManager;
+import org.leolo.nrdatad.db.CORPUSDao;
import org.leolo.nrdatad.db.MetadataDao;
import org.mariadb.jdbc.MariaDbPoolDataSource;
@@ -56,11 +57,18 @@ public class DatabaseManager implements org.leolo.nrdatad.db.DatabaseManager{
@Override
public Connection getConnection() throws SQLException{
- return ds.getConnection();
+ Connection conn = ds.getConnection();
+ conn.setAutoCommit(false);
+ return conn;
}
@Override
public MetadataDao getMetadataDao() {
return new org.leolo.nrdatad.db.mariadb.MetadataDao(this);
}
+
+ @Override
+ public CORPUSDao getCORPUSDao() {
+ return new org.leolo.nrdatad.db.mariadb.CORPUSDao(this);
+ }
}
diff --git a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java
index 31ff47f..9a41943 100644
--- a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java
+++ b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java
@@ -2,10 +2,22 @@ package org.leolo.nrdatad.org.leolo.nrdatad.cron;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.leolo.nrdatad.ConfigurationManager;
+import org.leolo.nrdatad.Constants;
+import org.leolo.nrdatad.db.CORPUSDao;
+import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS;
+import org.leolo.nrdatad.org.leolo.nrdatad.util.HttpUtil;
import org.quartz.Job;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+
public class ReferenceDataJob implements Job {
Logger log = LogManager.getLogger();
@@ -13,5 +25,48 @@ public class ReferenceDataJob implements Job {
@Override
public void execute(JobExecutionContext context) throws JobExecutionException {
log.atInfo().log("Load reference data triggered at {}", context.getFireTime());
+ //There are 3 different kind of reference data. Create 3 thread and deal with them
+ new Thread(()->{processCORPUS();}).start();
+ }
+
+ private void processCORPUS(){
+ ConfigurationManager conf = ConfigurationManager.getInstance();
+ log.atInfo().log("Loading CORPUS");
+ String json = null;
+ try {
+ json = HttpUtil.sendHttpRequestForGZipFile(
+ Constants.NetworkRailURI.CORPUS_URL,
+ conf.getProperty(Constants.Configuration.NETWORK_RAIL_USER),
+ conf.getProperty(Constants.Configuration.NETWORK_RAIL_PASSWORD)
+ );
+ } catch (IOException|URISyntaxException e) {
+ log.atError().withThrowable(e).log("Error when getting CORPUS data");
+ return;
+ }
+ JSONObject rootObj = new JSONObject(json);
+ JSONArray rootArray = rootObj.getJSONArray("TIPLOCDATA");
+ log.atInfo().log("{} entries found", rootArray.length());
+ ArrayList corpusList = new ArrayList<>();
+ for(int i=0;i