From 1148a15f1ae5af72bf5882b473110f2309122e75 Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Sun, 23 Oct 2022 17:43:31 +0100 Subject: [PATCH] Extra function for handling CORPUS --- src/main/java/org/leolo/nrdatad/db/CORPUSDao.java | 25 +++ .../org/leolo/nrdatad/db/mariadb/CORPUSDao.java | 242 ++++++++++++++++++++- .../org/leolo/nrdatad/cron/ReferenceDataJob.java | 3 +- .../nrdatad/org/leolo/nrdatad/model/CORPUS.java | 15 ++ 4 files changed, 279 insertions(+), 6 deletions(-) diff --git a/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java b/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java index 77d87ed..6e60e7e 100644 --- a/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java +++ b/src/main/java/org/leolo/nrdatad/db/CORPUSDao.java @@ -4,6 +4,7 @@ import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS; import java.sql.SQLException; import java.util.Collection; +import java.util.List; public abstract class CORPUSDao extends BaseDao{ @@ -26,4 +27,28 @@ public abstract class CORPUSDao extends BaseDao{ } public abstract void truncateTable() throws SQLException; + public abstract boolean hasNlcCode(String nlcCode) throws SQLException; + public abstract void update(CORPUS corpus) throws SQLException; + public void replace(CORPUS corpus) throws SQLException{ + if(corpus.getNlcCode()==null || !hasNlcCode(corpus.getNlcCode())){ + add(corpus); + }else{ + update(corpus); + } + } + + public void replaceAll(Collection datas) throws SQLException{ + for(CORPUS corpus: datas){ + replace(corpus); + } + } + + public abstract Collection searchCORPUSByStanoxCode(String stanoxCode) throws SQLException; + public abstract Collection searchCORPUSByUicCode(String uicCode) throws SQLException; + public abstract Collection searchCORPUSByCrsCode(String crsCode) throws SQLException; + public abstract Collection searchCORPUSByTiplocCode(String tiplocCode) throws SQLException; + public abstract CORPUS searchCORPUSByNlcCode(String nlcCode) throws SQLException; + public abstract List searchCORPUSByName(String name) throws SQLException; + + public abstract CORPUS delete(String nlcCode) throws SQLException; } diff --git a/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java b/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java index 4d9c847..75550bd 100644 --- a/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java +++ b/src/main/java/org/leolo/nrdatad/db/mariadb/CORPUSDao.java @@ -1,17 +1,21 @@ package org.leolo.nrdatad.db.mariadb; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jetbrains.annotations.NotNull; import org.leolo.nrdatad.db.DatabaseManager; import org.leolo.nrdatad.org.leolo.nrdatad.model.CORPUS; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.SQLException; -import java.sql.Statement; +import java.sql.*; import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Vector; public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao { + Logger log = LogManager.getLogger(); + public CORPUSDao(DatabaseManager manager) { super(manager); } @@ -64,6 +68,65 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao { } } + + @Override + public void replaceAll(@NotNull Collection datas) throws SQLException { + int totalCount = 0, insertCount = 0, updateCount = 0; + HashSet pendingInsert = new HashSet<>(); + try( + Connection conn = getConnection(); + PreparedStatement psIns = conn.prepareStatement( + "INSERT INTO corpus (stanox, uic_code, crs_code, tiploc_code, `desc`, short_desc, nlc_code) " + + "VALUES (?,?,?,?,?,?,?)"); + PreparedStatement psUpd = conn.prepareStatement( + "UPDATE corpus " + + "SET stanox=?, uic_code=?, crs_code=?, tiploc_code=?, `desc`=?, short_desc=? " + + "WHERE nlc_code = ?" + ) + ){ + for(CORPUS corpus:datas){ + totalCount++; + CORPUS existing = searchCORPUSByNlcCode(corpus.getNlcCode()); + PreparedStatement stmt = null; + if(corpus.equals(existing)){ + continue; + }else if(existing==null){ + if(pendingInsert.contains(corpus.getNlcCode())){ + continue; + } + pendingInsert.add(corpus.getNlcCode()); + stmt = psIns; + insertCount++; + }else{ + stmt = psUpd; + updateCount++; + } + setString(stmt, 1, corpus.getStanoxCode()); + setString(stmt, 2, corpus.getUicCode()); + setString(stmt, 3, corpus.getCrsCode()); + setString(stmt, 4, corpus.getTiplocCode()); + setString(stmt, 5, corpus.getLongDescription()); + setString(stmt, 6, corpus.getShortDescription()); + setString(stmt, 7, corpus.getNlcCode()); + stmt.addBatch(); + } + if(insertCount>0){ + psIns.executeBatch(); + log.atInfo().log("Inserted {} CORPUS records", insertCount); + } + if(updateCount>0){ + psUpd.executeBatch(); + log.atInfo().log("Updated {} CORPUS records", updateCount); + } + if(insertCount==0 && updateCount==0){ + log.atInfo().log("CORPUS data already up to date. "); + }else{ + conn.commit(); + log.atInfo().log("Done committing CORPUS data"); + } + } + } + @Override public void truncateTable() throws SQLException { try( @@ -73,4 +136,175 @@ public class CORPUSDao extends org.leolo.nrdatad.db.CORPUSDao { stmt.executeQuery("TRUNCATE TABLE corpus"); } } + + @Override + public boolean hasNlcCode(String nlcCode) throws SQLException { + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement("SELECT 1 FROM corpus WHERE nlc_code = ?") + ){ + pstmt.setString(1, nlcCode); + try(ResultSet rs = pstmt.executeQuery()){ + return rs.next(); + } + } + } + + @Override + public void update(CORPUS corpus) throws SQLException { + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + "UPDATE corpus " + + "SET stanox=?, uic_code=?, crs_code=?, tiploc_code=?, `desc`=?, short_desc=? " + + "WHERE nlc_code = ?" + ) + ){ + setString(pstmt, 1, corpus.getStanoxCode()); + setString(pstmt, 2, corpus.getUicCode()); + setString(pstmt, 3, corpus.getCrsCode()); + setString(pstmt, 4, corpus.getTiplocCode()); + setString(pstmt, 5, corpus.getLongDescription()); + setString(pstmt, 6, corpus.getShortDescription()); + setString(pstmt, 7, corpus.getNlcCode()); + pstmt.executeUpdate(); + } + } + + @Override + public Collection searchCORPUSByStanoxCode(String stanoxCode) throws SQLException { + Collection corpuses = new Vector<>(); + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM corpus WHERE stanox = ?") + ){ + pstmt.setString(1, stanoxCode); + try(ResultSet rs = pstmt.executeQuery()){ + while(rs.next()){ + corpuses.add(parseCORPUS(rs)); + } + } + } + return corpuses; + } + + @Override + public Collection searchCORPUSByUicCode(String uicCode) throws SQLException { + Collection corpuses = new Vector<>(); + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM corpus WHERE uic_code = ?") + ){ + pstmt.setString(1, uicCode); + try(ResultSet rs = pstmt.executeQuery()){ + while(rs.next()){ + corpuses.add(parseCORPUS(rs)); + } + } + } + return corpuses; + } + + @Override + public Collection searchCORPUSByCrsCode(String crsCode) throws SQLException { + Collection corpuses = new Vector<>(); + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM corpus WHERE crs_code = ?") + ){ + pstmt.setString(1, crsCode); + try(ResultSet rs = pstmt.executeQuery()){ + while(rs.next()){ + corpuses.add(parseCORPUS(rs)); + } + } + } + return corpuses; + } + + @Override + public Collection searchCORPUSByTiplocCode(String tiplocCode) throws SQLException { + Collection corpuses = new Vector<>(); + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM corpus WHERE tiploc_code = ?") + ){ + pstmt.setString(1, tiplocCode); + try(ResultSet rs = pstmt.executeQuery()){ + while(rs.next()){ + corpuses.add(parseCORPUS(rs)); + } + } + } + return corpuses; + } + + @Override + public CORPUS searchCORPUSByNlcCode(String nlcCode) throws SQLException { + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement("SELECT * FROM corpus WHERE nlc_code = ?") + ){ + pstmt.setString(1, nlcCode); + try(ResultSet rs = pstmt.executeQuery()){ + if(rs.next()){ + return parseCORPUS(rs); + } + } + } + return null; + } + + @Override + public List searchCORPUSByName(String name) throws SQLException { + + List corpuses = new Vector<>(); + try( + Connection conn = getConnection(); + PreparedStatement pstmt = conn.prepareStatement( + "SELECT * FROM corpus WHERE " + + "match(`desc`) against (?) OR " + + "match(short_desc) against (?) " + + "ORDER BY match(`desc`) against (?)+match(short_desc) against (?)" + ) + ){ + pstmt.setString(1, name); + pstmt.setString(2, name); + pstmt.setString(3, name); + pstmt.setString(4, name); + try(ResultSet rs = pstmt.executeQuery()){ + while(rs.next()){ + corpuses.add(parseCORPUS(rs)); + } + } + } + return corpuses; + } + + private CORPUS parseCORPUS(ResultSet rs) throws SQLException{ + CORPUS corpus = new CORPUS(); + corpus.setStanoxCode(rs.getString("stanox")); + corpus.setUicCode(rs.getString("uic_code")); + corpus.setCrsCode(rs.getString("crs_code")); + corpus.setTiplocCode(rs.getString("tiploc_code")); + corpus.setNlcCode(rs.getString("nlc_code")); + corpus.setLongDescription(rs.getString("desc")); + corpus.setShortDescription(rs.getString("short_desc")); + return corpus; + } + + @Override + public CORPUS delete(String nlcCode) throws SQLException { + CORPUS deletedEntry = searchCORPUSByNlcCode(nlcCode); + if(deletedEntry!=null){ + try( + Connection conn = getConnection(); + PreparedStatement stmt = conn.prepareStatement("DELETE FROM corpus WHERE nlc_code = ?") + ){ + stmt.setString(1, nlcCode); + stmt.executeUpdate(); + } + } + return deletedEntry; + } } diff --git a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java index 9a41943..b9dbc04 100644 --- a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java +++ b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/cron/ReferenceDataJob.java @@ -62,8 +62,7 @@ public class ReferenceDataJob implements Job { log.atInfo().log("Going to replace the data"); try { CORPUSDao cDao = conf.getDatabaseManager().getCORPUSDao(); - cDao.truncateTable(); - cDao.addAll(corpusList); + cDao.replaceAll(corpusList); } catch (SQLException e) { log.atError().withThrowable(e).log("Unable to insert records"); } diff --git a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/model/CORPUS.java b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/model/CORPUS.java index 4eba6e6..0b84652 100644 --- a/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/model/CORPUS.java +++ b/src/main/java/org/leolo/nrdatad/org/leolo/nrdatad/model/CORPUS.java @@ -1,5 +1,7 @@ package org.leolo.nrdatad.org.leolo.nrdatad.model; +import java.util.Objects; + public class CORPUS { private String stanoxCode; @@ -66,4 +68,17 @@ public class CORPUS { public void setShortDescription(String shortDescription) { this.shortDescription = shortDescription; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + CORPUS corpus = (CORPUS) o; + return Objects.equals(stanoxCode, corpus.stanoxCode) && Objects.equals(uicCode, corpus.uicCode) && Objects.equals(crsCode, corpus.crsCode) && Objects.equals(tiplocCode, corpus.tiplocCode) && Objects.equals(nlcCode, corpus.nlcCode) && Objects.equals(longDescription, corpus.longDescription) && Objects.equals(shortDescription, corpus.shortDescription); + } + + @Override + public int hashCode() { + return Objects.hash(stanoxCode, uicCode, crsCode, tiplocCode, nlcCode, longDescription, shortDescription); + } }