Processing TIPLOCs

src/main/java/org/leolo/nrdatad/cron/ScheduleImportJob.java

@@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger;
 import org.json.JSONObject;
 import org.leolo.nrdatad.ConfigurationManager;
 import org.leolo.nrdatad.Constants;
+import org.leolo.nrdatad.model.Tiploc;
 import org.leolo.nrdatad.util.HttpUtil;
 import org.quartz.Job;
 import org.quartz.JobExecutionContext;
@@ -12,9 +13,12 @@ import org.quartz.JobExecutionException;
 
 import java.io.*;
 import java.net.URISyntaxException;
+import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 public class ScheduleImportJob implements Job {
@@ -125,6 +129,48 @@ public class ScheduleImportJob implements Job {
 
     private void processTiploc(String file){
         log.atDebug().log("Processing {}",file);
-
+        try(
+                BufferedReader br = new BufferedReader(
+                        new InputStreamReader(
+                                new GZIPInputStream(
+                                        new FileInputStream(
+                                                new File(baseTempDir, file)
+                                        )
+                                )
+                        )
+                )
+        ){
+            int count = 0;
+            ArrayList<Tiploc> tiplocs = new ArrayList<>();
+            while(true){
+                String line = br.readLine();
+                if(line == null){
+                    break;
+                }
+                tiplocs.add(Tiploc.parseJSON(line));
+                if(tiplocs.size()>1000){
+                    try{
+                        ConfigurationManager.getInstance().getDatabaseManager().getTiplocDao().replaceAll(tiplocs);
+                    } catch (SQLException e){
+                        log.atWarn().withThrowable(e).log("Unable to update records.");
+                    }
+                    log.atDebug().log("Processed a batch of TIPLOCs");
+                    tiplocs.clear();
+                }
+            }
+            try{
+                ConfigurationManager.getInstance().getDatabaseManager().getTiplocDao().replaceAll(tiplocs);
+            } catch (SQLException e){
+                log.atWarn().withThrowable(e).log("Unable to update records.");
+            }
+            log.atDebug().log("Processed last batch of TIPLOCs");
+        } catch (IOException e){
+            log.atError().withThrowable(e).log("Error when reading file.");
+        }
+        try{
+            new File(baseTempDir, file).delete();
+        } catch (Exception e){
+            log.atWarn().withThrowable(e).log("Unable to remove temp file {}", file);
+        }
     }
 }