diff --git a/Jenkinsfile b/Jenkinsfile index c1f9ad7..d1cc0b4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,7 @@ pipeline{ } post{ success{ + junit 'target/surefire-reports/**/*.xml' archiveArtifacts artifacts: 'target/*.jar, *.pom', fingerprint: true, followSymlinks: false, onlyIfSuccessful: true } } diff --git a/pom.xml b/pom.xml index b58bf19..a1d9d3b 100644 --- a/pom.xml +++ b/pom.xml @@ -15,6 +15,19 @@ src/main/resources + + + org.apache.maven.plugins + maven-surefire-plugin + 3.0.0-M4 + + + org.leolo.map.osm.extract.test.** + + + + + 18 diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index 81a8589..4ad5749 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -7,6 +7,8 @@ import org.apache.commons.cli.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.leolo.map.osm.extract.model.*; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; @@ -58,6 +60,22 @@ public class ExtractElement { .required(true) .desc("output format") .build(); + + Option httpUserName = Option.builder() + .longOpt("http-user") + .argName("user name") + .hasArg(true) + .optionalArg(true) + .desc("User name sent during HTTP request") + .build(); + + Option httpPassword = Option.builder() + .longOpt("http-password") + .argName("passsword") + .hasArg(true) + .optionalArg(true) + .desc("Password sent during HTTP request") + .build(); } private Map allEntry = new Hashtable<>(); @@ -83,6 +101,8 @@ public class ExtractElement { String outputFilePath = null; String actionFilePath; String format; + InputFile dbFile; + InputFile actionFile; try { CommandLine cmd = parser.parse(getOptions(), args); if(args.length == 0 || cmd.hasOption(cliOpt.help)){ @@ -113,16 +133,8 @@ public class ExtractElement { System.exit(2); } PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath)); - File dbFile = new File(dbFilePath); - if(!dbFile.exists() || !dbFile.canRead()){ - log.atError().log("Unable to read db file {}", dbFilePath); - System.exit(2); - } - File actionFile = new File(actionFilePath); - if(!actionFile.exists() || !actionFile.canRead()){ - log.atError().log("Unable to read action file {}", actionFilePath); - System.exit(2); - } + dbFile = InputUtil.getInputFile(dbFilePath); + actionFile = InputUtil.getInputFile(actionFilePath); log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000); log.atInfo().log("Action file size {}kB", actionFile.length()/1_000); doExtract(dbFile, actionFile, outputStream); @@ -134,13 +146,13 @@ public class ExtractElement { System.err.println("Fatal error - unable to parse command line input!"); e.printStackTrace(); System.exit(1); - } catch (FileNotFoundException e) { + } catch (IOException e) { log.atError().withThrowable(e).log("Unable to locate the specified file"); System.exit(1); } } - private void doExtract(File dbFile, File actionFile, PrintStream outputStream) { + private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) { //Step 1: parse the actionFile ActionFile af; try { @@ -161,21 +173,21 @@ public class ExtractElement { * @param dbFile The database file going to be searched * @param af The file which defines the action to be performed */ - private void doSearch(File dbFile, ActionFile af) { + private void doSearch(InputFile dbFile, ActionFile af) { //Placeholder } - private void processNode(File dbFile) { + private void processNode(InputFile dbFile) { if(pendingNodes.isEmpty()) return; } - private void processWay(File dbFile) { + private void processWay(InputFile dbFile) { if(pendingWays.isEmpty()) return; } - private void expandRelations(File dbFile) { + private void expandRelations(InputFile dbFile) { if(pendingRelations.isEmpty()) return; } @@ -196,6 +208,9 @@ public class ExtractElement { options.addOption(cliOpt.actionFile); options.addOption(cliOpt.help); options.addOption(cliOpt.outputFormat); + //Commented first, need to find some way to pass the parameters to the function +// options.addOption(cliOpt.httpUserName); +// options.addOption(cliOpt.httpPassword); } return options; } diff --git a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java index 6179d27..e5d9f34 100644 --- a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java +++ b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java @@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.leolo.map.osm.extract.util.InputFile; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -23,10 +24,10 @@ public class ActionFile { private Map way = new Hashtable<>(); private Map node = new Hashtable<>(); - public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException { - log.atInfo().log("Parsing action file {}", path.getAbsolutePath()); + public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException { + log.atInfo().log("Parsing action file"); DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Document doc = builder.parse(path); + Document doc = builder.parse(path.getInputStream()); doc.getDocumentElement().normalize(); Element rootElement = doc.getDocumentElement(); log.atDebug().log("Root element : {}", rootElement.getTagName()); diff --git a/src/main/java/org/leolo/map/osm/extract/model/InputType.java b/src/main/java/org/leolo/map/osm/extract/model/InputType.java new file mode 100644 index 0000000..bb3e99d --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/InputType.java @@ -0,0 +1,6 @@ +package org.leolo.map.osm.extract.model; + +public enum InputType { + FILE, + URL; +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputFile.java b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java new file mode 100644 index 0000000..9c42ab5 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java @@ -0,0 +1,92 @@ +package org.leolo.map.osm.extract.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URL; + +public abstract class InputFile { + public abstract long getSize(); + public abstract InputStream getInputStream() throws IOException; + + public long length() { + return getSize(); + } + + public boolean isValid(){ + return false; + } +} + +class FileInputFile extends InputFile{ + + private File f; + + FileInputFile(File f){ + this.f = f; + } + + FileInputFile(String f){ + this.f = new File(f); + } + + @Override + public long getSize(){ + return f.length(); + } + + @Override + public InputStream getInputStream() throws FileNotFoundException { + return new FileInputStream(f); + } + + @Override + public boolean isValid(){ + return f.exists() && f.canRead(); + } +} + +class URLInputFile extends InputFile{ + + private Logger log = LogManager.getLogger(); + private boolean ready = false; + + private byte[] data; + + public static final int READ_BLOCK_SIZE = 4096; + + URLInputFile(String url) throws MalformedURLException, IOException { + this(new URL(url)); + } + URLInputFile(URL url) throws IOException{ + log.atInfo().log("Connecting to {}", url); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InputStream is = url.openStream(); + byte [] buffer = new byte[READ_BLOCK_SIZE]; + int readSize; + while((readSize = is.read(buffer))>0){ + baos.write(buffer, 0, readSize); + } + data = baos.toByteArray(); + is.close(); + baos.close(); + ready = true; + } + + @Override + public long getSize(){ + return data.length; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(data); + } + + @Override + public boolean isValid() { + return ready; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java new file mode 100644 index 0000000..8f19f7c --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java @@ -0,0 +1,27 @@ +package org.leolo.map.osm.extract.util; + +import org.leolo.map.osm.extract.model.InputType; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Locale; + +public class InputUtil { + public static InputType identifyInputType(String str){ + try { + URL url = new URL(str); + } catch (Exception e) { + return InputType.FILE; + } + return InputType.URL; + } + + public static InputFile getInputFile(String path) throws IOException { + if(identifyInputType(path)==InputType.FILE){ + return new FileInputFile(path); + }else{ + return new URLInputFile(path); + } + } +} diff --git a/src/test/java/MainTest.java b/src/test/java/MainTest.java deleted file mode 100644 index ca83c63..0000000 --- a/src/test/java/MainTest.java +++ /dev/null @@ -1,14 +0,0 @@ -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; - -import static org.junit.jupiter.api.Assertions.*; - -public class MainTest { - - @Test void test1(){ - - } - -} diff --git a/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java b/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java new file mode 100644 index 0000000..e451e5f --- /dev/null +++ b/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java @@ -0,0 +1,50 @@ +package org.leolo.map.osm.extract.test; + +import org.junit.jupiter.api.Test; +import org.leolo.map.osm.extract.model.InputType; +import org.leolo.map.osm.extract.util.InputUtil; + +import static org.junit.jupiter.api.Assertions.*; + +public class InputTypeTest { + + @Test void testFilePath(){ + assertEquals(InputUtil.identifyInputType("abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("../abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("./abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("..\\abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("/tmp/abc.xml"), InputType.FILE); + } + + @Test void testUPCPath(){ + assertEquals(InputUtil.identifyInputType("\\\\somecomputer\\somefolder\\abc.xml"), InputType.FILE); + } + + @Test void testURL(){ + assertEquals(InputUtil.identifyInputType("http://www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521/abc.xml"), InputType.URL); + } + +} diff --git a/src/test/java/org/leolo/map/osm/extract/test/MainTest.java b/src/test/java/org/leolo/map/osm/extract/test/MainTest.java new file mode 100644 index 0000000..79c71fa --- /dev/null +++ b/src/test/java/org/leolo/map/osm/extract/test/MainTest.java @@ -0,0 +1,23 @@ +package org.leolo.map.osm.extract.test; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; + +import java.io.IOException; +import java.util.ArrayList; + +import static org.junit.jupiter.api.Assertions.*; + +public class MainTest { + + @Test void test1(){ + System.out.println("Test executed."); + } + + @Test void downloadFile() throws IOException { + InputFile inputFile = InputUtil.getInputFile("https://www.leolo.pro/sample"); + assertEquals(inputFile.length(), 8192); + } +}