diff --git a/src/main/java/org/leolo/nrdatad/cron/ScheduleImportJob.java b/src/main/java/org/leolo/nrdatad/cron/ScheduleImportJob.java index ffe1736..740345c 100644 --- a/src/main/java/org/leolo/nrdatad/cron/ScheduleImportJob.java +++ b/src/main/java/org/leolo/nrdatad/cron/ScheduleImportJob.java @@ -5,6 +5,7 @@ import org.apache.logging.log4j.Logger; import org.json.JSONObject; import org.leolo.nrdatad.ConfigurationManager; import org.leolo.nrdatad.Constants; +import org.leolo.nrdatad.model.Tiploc; import org.leolo.nrdatad.util.HttpUtil; import org.quartz.Job; import org.quartz.JobExecutionContext; @@ -12,9 +13,12 @@ import org.quartz.JobExecutionException; import java.io.*; import java.net.URISyntaxException; +import java.sql.SQLException; +import java.util.ArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; public class ScheduleImportJob implements Job { @@ -125,6 +129,48 @@ public class ScheduleImportJob implements Job { private void processTiploc(String file){ log.atDebug().log("Processing {}",file); - + try( + BufferedReader br = new BufferedReader( + new InputStreamReader( + new GZIPInputStream( + new FileInputStream( + new File(baseTempDir, file) + ) + ) + ) + ) + ){ + int count = 0; + ArrayList tiplocs = new ArrayList<>(); + while(true){ + String line = br.readLine(); + if(line == null){ + break; + } + tiplocs.add(Tiploc.parseJSON(line)); + if(tiplocs.size()>1000){ + try{ + ConfigurationManager.getInstance().getDatabaseManager().getTiplocDao().replaceAll(tiplocs); + } catch (SQLException e){ + log.atWarn().withThrowable(e).log("Unable to update records."); + } + log.atDebug().log("Processed a batch of TIPLOCs"); + tiplocs.clear(); + } + } + try{ + ConfigurationManager.getInstance().getDatabaseManager().getTiplocDao().replaceAll(tiplocs); + } catch (SQLException e){ + log.atWarn().withThrowable(e).log("Unable to update records."); + } + log.atDebug().log("Processed last batch of TIPLOCs"); + } catch (IOException e){ + log.atError().withThrowable(e).log("Error when reading file."); + } + try{ + new File(baseTempDir, file).delete(); + } catch (Exception e){ + log.atWarn().withThrowable(e).log("Unable to remove temp file {}", file); + } } }