Will retrieve the file directly from network rail

.gitignore

@@ -1,3 +1,4 @@
 /bin/
 /target/
 /configuration.properties
+output.file

src/org/leolo/rail/nrd/Constants.java

@@ -1,6 +1,6 @@
 package org.leolo.rail.nrd;
 
 public class Constants {
-	public static final int BATCH_SIZE = 500;
+	public static final int BATCH_SIZE = 10000;
-	public static final int MAX_QUEUE_SIZE = 20000;
+	public static final int MAX_QUEUE_SIZE = 40000;
 }

src/org/leolo/rail/nrd/FileLoader.java

@@ -6,10 +6,16 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.Authenticator;
+import java.net.PasswordAuthentication;
+import java.net.URL;
+import java.net.URLConnection;
 import java.sql.Connection;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.sql.Statement;
+import java.util.Base64;
 import java.util.Hashtable;
 import java.util.Map;
 import java.util.Properties;
@@ -17,6 +23,7 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipException;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -50,13 +57,18 @@ public class FileLoader {
 		}
 		InputStream fis = null;
 		//TODO: get the file from Network Rail
-		
 		try {
-			fis = new FileInputStream(ConfigurationManager.getInstance().getProperty("file.path"));
+			URL url = new URL(ConfigurationManager.getInstance().getProperty("remote.path"));
+			URLConnection conn = url.openConnection();
+			String userpwd = ConfigurationManager.getInstance().getProperty("remote.user")+":"+ConfigurationManager.getInstance().getProperty("remote.pwd");
+			conn.addRequestProperty("Authorization", "Basic "+Base64.getEncoder().encodeToString(userpwd.getBytes()));
+			conn.connect();
+			fis = conn.getInputStream();
 		}catch(IOException e) {
-			log.fatal(e.getMessage(), e);
+			log.error(e.getMessage(), e);
-			System.exit(1);
+			System.exit(4);
 		}
+		log.info("Connected!");
 		DatabaseManager.getInstance().clear();
 		try(
 				GZIPInputStream gis = new GZIPInputStream(fis);
@@ -103,8 +115,12 @@ public class FileLoader {
 			asso.shutdownAndWait();
 			tiploc.shutdownAndWait();
 			schedule.shutdownAndWait();
+		}catch(ZipException e) {
+			log.error(e.getMessage(), e);
+			captureFile();
 		}catch(IOException e) {
 			log.error(e.getMessage(), e);
+//			log.error(e.)
 			System.exit(1);
 		}
 		try {
@@ -117,4 +133,23 @@ public class FileLoader {
 		log.info("Job finished!");
 	}
 
+	private void captureFile() {
+		try {
+			URL url = new URL(ConfigurationManager.getInstance().get("remote.path").toString());
+			BufferedReader br = new BufferedReader(new InputStreamReader(url.openStream()));
+			PrintWriter out = new PrintWriter("output.file");
+			while(true) {
+				String line = br.readLine();
+				if(line==null) {
+					break;
+				}
+				out.println(line);
+			}
+			out.close();
+			br.close();
+		}catch(IOException e) {
+			log.error(e.getMessage(), e);
+		}
+	}
+
 }