diff --git a/pom.xml b/pom.xml index db931e6..3ee36d2 100644 --- a/pom.xml +++ b/pom.xml @@ -44,12 +44,12 @@ org.mariadb.jdbc mariadb-java-client - 2.7.2 + 2.7.6 org.json json - 20210307 + 20220924 org.quartz-scheduler @@ -67,6 +67,11 @@ log4j-slf4j-impl ${ver-log4j2} + + org.json + json + 20220924 + @@ -119,8 +124,8 @@ org.apache.maven.plugins maven-compiler-plugin - 8 - 8 + 11 + 11 diff --git a/src/main/java/org/leolo/nrdatad/Constants.java b/src/main/java/org/leolo/nrdatad/Constants.java index c955249..07a083c 100644 --- a/src/main/java/org/leolo/nrdatad/Constants.java +++ b/src/main/java/org/leolo/nrdatad/Constants.java @@ -12,5 +12,11 @@ public final class Constants { public static class Configuration { public static final String ALWAYS_RUN_LTP = "cron.ltp.always"; public static final String ALWAYS_RUN_REF_DATA = "cron.ref.always"; + public static final String NETWORK_RAIL_USER = "network.user"; + public static final String NETWORK_RAIL_PASSWORD = "network.pwd"; + } + + public static class NetworkRailURI{ + public static final String CORPUS_URL = "https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS"; } } diff --git a/src/main/java/org/leolo/nrdatad/db/BaseDao.java b/src/main/java/org/leolo/nrdatad/db/BaseDao.java index 383a8a9..e78fd13 100644 --- a/src/main/java/org/leolo/nrdatad/db/BaseDao.java +++ b/src/main/java/org/leolo/nrdatad/db/BaseDao.java @@ -20,7 +20,7 @@ public abstract class BaseDao { } protected void setString(PreparedStatement stmt, int pos, @Nullable String data) throws SQLException{ - if(data==null){ + if(data==null||data.strip().length()==0){ stmt.setNull(pos, Types.CHAR); }else{ stmt.setString(pos, data); diff --git a/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java b/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java index 8c125b9..77d87ed 100644 --- a/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java +++ b/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java @@ -24,4 +24,6 @@ public abstract class CORPUSDao extends BaseDao{ add(corpus); } } + + public abstract void truncateTable() throws SQLException; } diff --git a/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java b/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java index b738397..f9b99bf 100644 --- a/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java +++ b/src/main/java/org/leolo/nrdatad/db/DatabaseManager.java @@ -13,4 +13,6 @@ public interface DatabaseManager { public Connection getConnection() throws SQLException; public MetadataDao getMetadataDao(); + + public CORPUSDao getCORPUSDao(); } diff --git a/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java b/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java index f1d13e1..4d9c847 100644 --- a/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java +++ b/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java @@ -7,6 +7,7 @@ import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Statement; import java.util.Collection; public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao { @@ -36,6 +37,7 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao { setString(pstmt, 6, corpus.getLongDescription()); setString(pstmt, 7, corpus.getShortDescription()); pstmt.executeUpdate(); + conn.commit(); } } @@ -58,6 +60,17 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao { pstmt.addBatch(); } pstmt.executeBatch(); + conn.commit(); + } + } + + @Override + public void truncateTable() throws SQLException { + try( + Connection conn = getConnection(); + Statement stmt = conn.createStatement() + ){ + stmt.executeQuery("TRUNCATE TABLE corpus"); } } } diff --git a/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java b/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java index d27e98f..0b88ba4 100644 --- a/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java +++ b/src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java @@ -3,6 +3,7 @@ package org.leolo.nrdatad.db.mariadb; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.leolo.nrdatad.ConfigurationManager; +import org.leolo.nrdatad.db.CORPUSDao; import org.leolo.nrdatad.db.MetadataDao; import org.mariadb.jdbc.MariaDbPoolDataSource; @@ -56,11 +57,18 @@ public class DatabaseManager implements org.leolo.nrdatad.db.DatabaseManager{ @Override public Connection getConnection() throws SQLException{ - return ds.getConnection(); + Connection conn = ds.getConnection(); + conn.setAutoCommit(false); + return conn; } @Override public MetadataDao getMetadataDao() { return new org.leolo.nrdatad.db.mariadb.MetadataDao(this); } + + @Override + public CORPUSDao getCORPUSDao() { + return new org.leolo.nrdatad.db.mariadb.CORPUSDao(this); + } } diff --git a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java index 31ff47f..9a41943 100644 --- a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java +++ b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java @@ -2,10 +2,22 @@ package org.leolo.nrdatad.org.leolo.nrdatad.cron; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.leolo.nrdatad.ConfigurationManager; +import org.leolo.nrdatad.Constants; +import org.leolo.nrdatad.db.CORPUSDao; +import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS; +import org.leolo.nrdatad.org.leolo.nrdatad.util.HttpUtil; import org.quartz.Job; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; +import java.io.IOException; +import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.ArrayList; + public class ReferenceDataJob implements Job { Logger log = LogManager.getLogger(); @@ -13,5 +25,48 @@ public class ReferenceDataJob implements Job { @Override public void execute(JobExecutionContext context) throws JobExecutionException { log.atInfo().log("Load reference data triggered at {}", context.getFireTime()); + //There are 3 different kind of reference data. Create 3 thread and deal with them + new Thread(()->{processCORPUS();}).start(); + } + + private void processCORPUS(){ + ConfigurationManager conf = ConfigurationManager.getInstance(); + log.atInfo().log("Loading CORPUS"); + String json = null; + try { + json = HttpUtil.sendHttpRequestForGZipFile( + Constants.NetworkRailURI.CORPUS_URL, + conf.getProperty(Constants.Configuration.NETWORK_RAIL_USER), + conf.getProperty(Constants.Configuration.NETWORK_RAIL_PASSWORD) + ); + } catch (IOException|URISyntaxException e) { + log.atError().withThrowable(e).log("Error when getting CORPUS data"); + return; + } + JSONObject rootObj = new JSONObject(json); + JSONArray rootArray = rootObj.getJSONArray("TIPLOCDATA"); + log.atInfo().log("{} entries found", rootArray.length()); + ArrayList corpusList = new ArrayList<>(); + for(int i=0;i