diff --git a/src/org/leolo/rail/nrd/AssoicationProcessor.java b/src/org/leolo/rail/nrd/AssoicationProcessor.java index 7d9e285..44b0be6 100644 --- a/src/org/leolo/rail/nrd/AssoicationProcessor.java +++ b/src/org/leolo/rail/nrd/AssoicationProcessor.java @@ -2,15 +2,18 @@ package org.leolo.rail.nrd; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.zip.GZIPInputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,7 +33,7 @@ public class AssoicationProcessor implements Runnable { log.info("Processing {}", fileName.getName()); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); try( - BufferedReader br = new BufferedReader(new FileReader(fileName)); + BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName)))); Connection conn = DatabaseManager.getInstance().getConnection(); PreparedStatement pstmt = conn.prepareStatement("INSERT INTO train_assoc VALUES(null,?,?,?,?,?,?,?,?,?,?,?,?)") ){ diff --git a/src/org/leolo/rail/nrd/FileLoader.java b/src/org/leolo/rail/nrd/FileLoader.java index f893ddf..643ed2b 100644 --- a/src/org/leolo/rail/nrd/FileLoader.java +++ b/src/org/leolo/rail/nrd/FileLoader.java @@ -3,10 +3,13 @@ package org.leolo.rail.nrd; import java.io.BufferedReader; import java.io.File; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.FileReader; +import java.io.FileWriter; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.net.URL; import java.net.URLConnection; @@ -14,6 +17,7 @@ import java.util.Base64; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import java.util.zip.ZipException; import org.apache.logging.log4j.LogManager; @@ -71,9 +75,9 @@ public class FileLoader { GZIPInputStream gis = new GZIPInputStream(fis); BufferedReader br = new BufferedReader(new InputStreamReader(gis)); ){ - PrintWriter ass = new PrintWriter(new File(tempDir, "ass_0")); - PrintWriter tip = new PrintWriter(new File(tempDir, "tip_0")); - PrintWriter sch = new PrintWriter(new File(tempDir, "sch_0")); + PrintWriter ass = new PrintWriter(new GZIPOutputStream(new FileOutputStream(new File(tempDir, "ass_0")))); + PrintWriter tip = new PrintWriter(new GZIPOutputStream(new FileOutputStream(new File(tempDir, "tip_0")))); + PrintWriter sch = new PrintWriter(new GZIPOutputStream(new FileOutputStream(new File(tempDir, "sch_0")))); int count = 0; while(true) { String line = br.readLine(); @@ -90,21 +94,21 @@ public class FileLoader { countA++; if(countA%Constants.BATCH_SIZE==0) { ass.close(); - ass = new PrintWriter(new File(tempDir, "ass_"+(++batchA))); + ass = new PrintWriter(new GZIPOutputStream(new FileOutputStream(new File(tempDir, "ass_"+(++batchA))))); } }else if("TiplocV1".equals(objectType)){ tip.println(obj.getJSONObject("TiplocV1")); countT++; if(countT%Constants.BATCH_SIZE==0) { tip.close(); - tip = new PrintWriter(new File(tempDir, "tip_"+(++batchT))); + tip = new PrintWriter(new GZIPOutputStream(new FileOutputStream(new File(tempDir, "tip_"+(++batchT))))); } }else if("JsonScheduleV1".equals(objectType)){ sch.println(obj.getJSONObject("JsonScheduleV1")); countS++; if(countS%Constants.BATCH_SIZE==0) { sch.close(); - sch = new PrintWriter(new File(tempDir, "sch_"+(++batchS))); + sch = new PrintWriter(new GZIPOutputStream(new FileOutputStream(new File(tempDir, "sch_"+(++batchS))))); } }else if("EOF".equals(objectType)){ //Nothing to do diff --git a/src/org/leolo/rail/nrd/ScheduleProcessor.java b/src/org/leolo/rail/nrd/ScheduleProcessor.java index 28b450d..ad7c102 100644 --- a/src/org/leolo/rail/nrd/ScheduleProcessor.java +++ b/src/org/leolo/rail/nrd/ScheduleProcessor.java @@ -2,9 +2,11 @@ package org.leolo.rail.nrd; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; @@ -12,6 +14,7 @@ import java.sql.Time; import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.zip.GZIPInputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -79,7 +82,7 @@ public class ScheduleProcessor implements Runnable { log.info("Processing {}", fileName.getName()); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd"); try( - BufferedReader br = new BufferedReader(new FileReader(fileName)); + BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName)))); Connection conn = DatabaseManager.getInstance().getConnection(); PreparedStatement sMain = conn.prepareStatement("INSERT INTO train_schedule VALUES (?,?,?,?,?,?,?)"); PreparedStatement sDetail = conn.prepareStatement("INSERT INTO train_schedule_detail VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"); diff --git a/src/org/leolo/rail/nrd/TiplocProcessor.java b/src/org/leolo/rail/nrd/TiplocProcessor.java index 63672f6..02ace43 100644 --- a/src/org/leolo/rail/nrd/TiplocProcessor.java +++ b/src/org/leolo/rail/nrd/TiplocProcessor.java @@ -2,15 +2,18 @@ package org.leolo.rail.nrd; import java.io.BufferedReader; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.IOException; +import java.io.InputStreamReader; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.SQLException; import java.sql.Types; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.zip.GZIPInputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -30,7 +33,7 @@ public class TiplocProcessor implements Runnable { log.info("Processing {}", fileName.getName()); SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); try( - BufferedReader br = new BufferedReader(new FileReader(fileName)); + BufferedReader br = new BufferedReader(new InputStreamReader(new GZIPInputStream(new FileInputStream(fileName)))); Connection conn = DatabaseManager.getInstance().getConnection(); PreparedStatement pstmt = conn.prepareStatement("INSERT INTO tiploc VALUES (?,?,?,?,?,?)") ){