Job to insert the records.

pom.xml

@@ -44,12 +44,12 @@
     <dependency>
       <groupId>org.mariadb.jdbc</groupId>
       <artifactId>mariadb-java-client</artifactId>
-      <version>2.7.2</version>
+      <version>2.7.6</version>
     </dependency>
     <dependency>
       <groupId>org.json</groupId>
       <artifactId>json</artifactId>
-      <version>20210307</version>
+      <version>20220924</version>
     </dependency>
     <dependency>
       <groupId>org.quartz-scheduler</groupId>
@@ -67,6 +67,11 @@
     <artifactId>log4j-slf4j-impl</artifactId>
     <version>${ver-log4j2}</version>
   </dependency>
+    <dependency>
+      <groupId>org.json</groupId>
+      <artifactId>json</artifactId>
+      <version>20220924</version>
+    </dependency>
 
   </dependencies>
 
@@ -119,8 +124,8 @@
         <groupId>org.apache.maven.plugins</groupId>
         <artifactId>maven-compiler-plugin</artifactId>
         <configuration>
-          <source>8</source>
-          <target>8</target>
+          <source>11</source>
+          <target>11</target>
         </configuration>
       </plugin>
     </plugins>

src/main/java/org/leolo/nrdatad/Constants.java

@@ -12,5 +12,11 @@ public final class Constants {
     public static class Configuration {
         public static final String ALWAYS_RUN_LTP = "cron.ltp.always";
         public static final String ALWAYS_RUN_REF_DATA = "cron.ref.always";
+        public static final String NETWORK_RAIL_USER = "network.user";
+        public static final String NETWORK_RAIL_PASSWORD = "network.pwd";
+    }
+
+    public static class NetworkRailURI{
+        public static final String CORPUS_URL = "https://datafeeds.networkrail.co.uk/ntrod/SupportingFileAuthenticate?type=CORPUS";
     }
 }

src/main/java/org/leolo/nrdatad/db/BaseDao.java

@@ -20,7 +20,7 @@ public abstract class BaseDao {
     }
 
     protected void setString(PreparedStatement stmt, int pos, @Nullable String data) throws SQLException{
-        if(data==null){
+        if(data==null||data.strip().length()==0){
             stmt.setNull(pos, Types.CHAR);
         }else{
             stmt.setString(pos, data);

src/main/java/org/leolo/nrdatad/db/CORPUSDao.java

@@ -24,4 +24,6 @@ public abstract class CORPUSDao extends BaseDao{
             add(corpus);
         }
     }
+
+    public abstract void truncateTable() throws SQLException;
 }

src/main/java/org/leolo/nrdatad/db/DatabaseManager.java

@@ -13,4 +13,6 @@ public interface DatabaseManager {
     public Connection getConnection() throws SQLException;
 
     public MetadataDao getMetadataDao();
+
+    public CORPUSDao getCORPUSDao();
 }

src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java

@@ -7,6 +7,7 @@ import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.sql.Statement;
 import java.util.Collection;
 
 public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao {
@@ -36,6 +37,7 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao {
             setString(pstmt, 6, corpus.getLongDescription());
             setString(pstmt, 7, corpus.getShortDescription());
             pstmt.executeUpdate();
+            conn.commit();
         }
     }
 
@@ -58,6 +60,17 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao {
                 pstmt.addBatch();
             }
             pstmt.executeBatch();
+            conn.commit();
+        }
+    }
+
+    @Override
+    public void truncateTable() throws SQLException {
+        try(
+                Connection conn = getConnection();
+                Statement stmt = conn.createStatement()
+        ){
+            stmt.executeQuery("TRUNCATE TABLE corpus");
         }
     }
 }

src/main/java/org/leolo/nrdatad/db/mariadb/DatabaseManager.java

@@ -3,6 +3,7 @@ package org.leolo.nrdatad.db.mariadb;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.leolo.nrdatad.ConfigurationManager;
+import org.leolo.nrdatad.db.CORPUSDao;
 import org.leolo.nrdatad.db.MetadataDao;
 import org.mariadb.jdbc.MariaDbPoolDataSource;
 
@@ -56,11 +57,18 @@ public class DatabaseManager implements org.leolo.nrdatad.db.DatabaseManager{
 
     @Override
     public Connection getConnection() throws SQLException{
-        return ds.getConnection();
+        Connection conn = ds.getConnection();
+        conn.setAutoCommit(false);
+        return conn;
     }
 
     @Override
     public MetadataDao getMetadataDao() {
         return new org.leolo.nrdatad.db.mariadb.MetadataDao(this);
     }
+
+    @Override
+    public CORPUSDao getCORPUSDao() {
+        return new org.leolo.nrdatad.db.mariadb.CORPUSDao(this);
+    }
 }

src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java

@@ -2,10 +2,22 @@ package org.leolo.nrdatad.org.leolo.nrdatad.cron;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.leolo.nrdatad.ConfigurationManager;
+import org.leolo.nrdatad.Constants;
+import org.leolo.nrdatad.db.CORPUSDao;
+import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS;
+import org.leolo.nrdatad.org.leolo.nrdatad.util.HttpUtil;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
 import org.quartz.JobExecutionException;
 
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.sql.SQLException;
+import java.util.ArrayList;
+
 public class ReferenceDataJob implements Job {
 
     Logger log = LogManager.getLogger();
@@ -13,5 +25,48 @@ public class ReferenceDataJob implements Job {
     @Override
     public void execute(JobExecutionContext context) throws JobExecutionException {
         log.atInfo().log("Load reference data triggered at {}", context.getFireTime());
+        //There are 3 different kind of reference data. Create 3 thread and deal with them
+        new Thread(()->{processCORPUS();}).start();
+    }
+
+    private void processCORPUS(){
+        ConfigurationManager conf = ConfigurationManager.getInstance();
+        log.atInfo().log("Loading CORPUS");
+        String json = null;
+        try {
+            json = HttpUtil.sendHttpRequestForGZipFile(
+                    Constants.NetworkRailURI.CORPUS_URL,
+                    conf.getProperty(Constants.Configuration.NETWORK_RAIL_USER),
+                    conf.getProperty(Constants.Configuration.NETWORK_RAIL_PASSWORD)
+            );
+        } catch (IOException|URISyntaxException e) {
+            log.atError().withThrowable(e).log("Error when getting CORPUS data");
+            return;
+        }
+        JSONObject rootObj = new JSONObject(json);
+        JSONArray rootArray = rootObj.getJSONArray("TIPLOCDATA");
+        log.atInfo().log("{} entries found", rootArray.length());
+        ArrayList<CORPUS> corpusList = new ArrayList<>();
+        for(int i=0;i<rootArray.length();i++){
+            JSONObject obj = rootArray.getJSONObject(i);
+            CORPUS corpus = new CORPUS();
+            corpus.setStanoxCode(obj.optString("STANOX"));
+            corpus.setUicCode(obj.optString("UIC"));
+            corpus.setCrsCode(obj.optString("3ALPHA"));
+            corpus.setTiplocCode(obj.optString("TIPLOC"));
+            corpus.setNlcCode(obj.optString("NLC"));
+            corpus.setLongDescription(obj.optString("NLCDESC"));
+            corpus.setShortDescription(obj.optString("NLCDESC16"));
+            corpusList.add(corpus);
+        }
+        log.atInfo().log("Going to replace the data");
+        try {
+            CORPUSDao cDao = conf.getDatabaseManager().getCORPUSDao();
+            cDao.truncateTable();
+            cDao.addAll(corpusList);
+        } catch (SQLException e) {
+            log.atError().withThrowable(e).log("Unable to insert records");
+        }
+        log.atInfo().log("Done inserting CORPUS");
     }
 }

src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/util/HttpUtil.java

@@ -0,0 +1,63 @@
+package org.leolo.nrdatad.org.leolo.nrdatad.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.leolo.nrdatad.ConfigurationManager;
+import org.leolo.nrdatad.Constants;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Base64;
+import java.util.zip.GZIPInputStream;
+
+public class HttpUtil {
+
+    Logger log = LogManager.getLogger();
+
+    public static String sendHttpRequest(URL url, String userName, String password) throws IOException {
+        URLConnection conn = url.openConnection();
+        String userpwd = userName+":"+password;
+        conn.addRequestProperty("Authorization", "Basic "+ Base64.getEncoder().encodeToString(userpwd.getBytes()));
+        conn.connect();
+        StringBuilder sb = new StringBuilder();
+        try(BufferedReader br = new BufferedReader(new InputStreamReader(conn.getInputStream()))){
+            while(true){
+                String line = br.readLine();
+                if(line == null) break;
+                sb.append(line).append("\n");
+            }
+        }
+        return sb.toString();
+    }
+
+    public static String sendHttpRequest(String url, String userName, String password) throws IOException, URISyntaxException {
+        return sendHttpRequest(new URL(url), userName, password);
+    }
+    public static String sendHttpRequestForGZipFile(URL url, String userName, String password) throws IOException {
+        URLConnection conn = url.openConnection();
+        String userpwd = userName+":"+password;
+        conn.addRequestProperty("Authorization", "Basic "+ Base64.getEncoder().encodeToString(userpwd.getBytes()));
+        conn.connect();
+        StringBuilder sb = new StringBuilder();
+        try(
+                GZIPInputStream gis = new GZIPInputStream(conn.getInputStream());
+                BufferedReader br = new BufferedReader(new InputStreamReader(gis))
+        ){
+            while(true){
+                String line = br.readLine();
+                if(line == null) break;
+                sb.append(line).append("\n");
+            }
+        }
+        return sb.toString();
+    }
+
+    public static String sendHttpRequestForGZipFile(String url, String userName, String password) throws IOException, URISyntaxException {
+        return sendHttpRequestForGZipFile(new URL(url), userName, password);
+    }
+
+}

src/test/java/org/leolo/nrdatad/db/test/DatabaseManager.java

@@ -1,5 +1,6 @@
 package org.leolo.nrdatad.db.test;
 
+import org.leolo.nrdatad.db.CORPUSDao;
 import org.leolo.nrdatad.db.MetadataDao;
 
 import java.sql.Connection;
@@ -25,4 +26,9 @@ public class DatabaseManager implements org.leolo.nrdatad.db.DatabaseManager {
     public MetadataDao getMetadataDao() {
         return null;
     }
+
+    @Override
+    public CORPUSDao getCORPUSDao() {
+        return null;
+    }
 }