Merge branch 'develop' into feature/de.topobyte-lib

# Conflicts:
#	src/main/java/org/leolo/map/osm/extract/ExtractElement.java

Jenkinsfile

@@ -21,6 +21,7 @@ pipeline{
             }
             post{
                 success{
+                    junit 'target/surefire-reports/**/*.xml'
                     archiveArtifacts artifacts: 'target/*.jar, *.pom', fingerprint: true, followSymlinks: false, onlyIfSuccessful: true
                 }
             }

pom.xml

@@ -15,6 +15,19 @@
                 <directory>src/main/resources</directory>
             </resource>
         </resources>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>3.0.0-M4</version>
+                <configuration>
+                    <includes>
+                        <include>org.leolo.map.osm.extract.test.**</include>
+
+                    </includes>
+                </configuration>
+            </plugin>
+        </plugins>
     </build>
     <properties>
         <maven.compiler.source>18</maven.compiler.source>

src/main/java/org/leolo/map/osm/extract/ExtractElement.java

@@ -7,6 +7,8 @@ import org.apache.commons.cli.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.leolo.map.osm.extract.model.*;
+import org.leolo.map.osm.extract.util.InputFile;
+import org.leolo.map.osm.extract.util.InputUtil;
 import org.xml.sax.SAXException;
 
 import javax.xml.parsers.ParserConfigurationException;
@@ -58,6 +60,22 @@ public class ExtractElement {
                 .required(true)
                 .desc("output format")
                 .build();
+
+        Option httpUserName = Option.builder()
+                .longOpt("http-user")
+                .argName("user name")
+                .hasArg(true)
+                .optionalArg(true)
+                .desc("User name sent during HTTP request")
+                .build();
+
+        Option httpPassword = Option.builder()
+                .longOpt("http-password")
+                .argName("passsword")
+                .hasArg(true)
+                .optionalArg(true)
+                .desc("Password sent during HTTP request")
+                .build();
     }
 
     private Map<Long, MapEntry> allEntry = new Hashtable<>();
@@ -83,6 +101,8 @@ public class ExtractElement {
         String outputFilePath = null;
         String actionFilePath;
         String format;
+        InputFile dbFile;
+        InputFile actionFile;
         try {
             CommandLine cmd = parser.parse(getOptions(), args);
             if(args.length == 0 || cmd.hasOption(cliOpt.help)){
@@ -113,16 +133,8 @@ public class ExtractElement {
                 System.exit(2);
             }
             PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath));
-            File dbFile = new File(dbFilePath);
+            dbFile = InputUtil.getInputFile(dbFilePath);
-            if(!dbFile.exists() || !dbFile.canRead()){
+            actionFile = InputUtil.getInputFile(actionFilePath);
-                log.atError().log("Unable to read db file {}", dbFilePath);
-                System.exit(2);
-            }
-            File actionFile = new File(actionFilePath);
-            if(!actionFile.exists() || !actionFile.canRead()){
-                log.atError().log("Unable to read action file {}", actionFilePath);
-                System.exit(2);
-            }
             log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000);
             log.atInfo().log("Action file size {}kB", actionFile.length()/1_000);
             doExtract(dbFile, actionFile, outputStream);
@@ -134,13 +146,13 @@ public class ExtractElement {
             System.err.println("Fatal error - unable to parse command line input!");
             e.printStackTrace();
             System.exit(1);
-        } catch (FileNotFoundException e) {
+        } catch (IOException e) {
             log.atError().withThrowable(e).log("Unable to locate the specified file");
             System.exit(1);
         }
     }
 
-    private void doExtract(File dbFile, File actionFile, PrintStream outputStream) {
+    private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) {
         //Step 1: parse the actionFile
         ActionFile af;
         try {
@@ -161,21 +173,21 @@ public class ExtractElement {
      * @param dbFile The database file going to be searched
      * @param af The file which defines the action to be performed
      */
-    private void doSearch(File dbFile, ActionFile af) {
+    private void doSearch(InputFile dbFile, ActionFile af) {
         //Placeholder
     }
 
-    private void processNode(File dbFile) {
+    private void processNode(InputFile dbFile) {
         if(pendingNodes.isEmpty())
             return;
     }
 
-    private void processWay(File dbFile) {
+    private void processWay(InputFile dbFile) {
         if(pendingWays.isEmpty())
             return;
     }
 
-    private void expandRelations(File dbFile) {
+    private void expandRelations(InputFile dbFile) {
         if(pendingRelations.isEmpty())
             return;
     }
@@ -196,6 +208,9 @@ public class ExtractElement {
             options.addOption(cliOpt.actionFile);
             options.addOption(cliOpt.help);
             options.addOption(cliOpt.outputFormat);
+            //Commented first, need to find some way to pass the parameters to the function
+//            options.addOption(cliOpt.httpUserName);
+//            options.addOption(cliOpt.httpPassword);
         }
         return options;
     }

src/main/java/org/leolo/map/osm/extract/model/ActionFile.java

@@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.leolo.map.osm.extract.util.InputFile;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -23,10 +24,10 @@ public class ActionFile {
     private Map<Long,ExtractItem> way = new Hashtable<>();
     private Map<Long,ExtractItem> node = new Hashtable<>();
 
-    public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException {
+    public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException {
-        log.atInfo().log("Parsing action file {}", path.getAbsolutePath());
+        log.atInfo().log("Parsing action file");
         DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-        Document doc = builder.parse(path);
+        Document doc = builder.parse(path.getInputStream());
         doc.getDocumentElement().normalize();
         Element rootElement = doc.getDocumentElement();
         log.atDebug().log("Root element : {}", rootElement.getTagName());

src/main/java/org/leolo/map/osm/extract/model/InputType.java

@@ -0,0 +1,6 @@
+package org.leolo.map.osm.extract.model;
+
+public enum InputType {
+    FILE,
+    URL;
+}

src/main/java/org/leolo/map/osm/extract/util/InputFile.java

@@ -0,0 +1,92 @@
+package org.leolo.map.osm.extract.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public abstract class InputFile {
+    public abstract long getSize();
+    public abstract InputStream getInputStream() throws IOException;
+
+    public long length() {
+        return getSize();
+    }
+
+    public boolean isValid(){
+        return false;
+    }
+}
+
+class FileInputFile extends InputFile{
+
+    private File f;
+
+    FileInputFile(File f){
+        this.f = f;
+    }
+
+    FileInputFile(String f){
+        this.f = new File(f);
+    }
+
+    @Override
+    public long getSize(){
+        return f.length();
+    }
+
+    @Override
+    public InputStream getInputStream() throws FileNotFoundException {
+        return new FileInputStream(f);
+    }
+
+    @Override
+    public boolean isValid(){
+        return f.exists() && f.canRead();
+    }
+}
+
+class URLInputFile extends InputFile{
+
+    private Logger log = LogManager.getLogger();
+    private boolean ready = false;
+
+    private byte[] data;
+
+    public static final int READ_BLOCK_SIZE = 4096;
+
+    URLInputFile(String url) throws MalformedURLException, IOException {
+        this(new URL(url));
+    }
+    URLInputFile(URL url) throws IOException{
+        log.atInfo().log("Connecting to {}", url);
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        InputStream is = url.openStream();
+        byte [] buffer = new byte[READ_BLOCK_SIZE];
+        int readSize;
+        while((readSize = is.read(buffer))>0){
+            baos.write(buffer, 0, readSize);
+        }
+        data = baos.toByteArray();
+        is.close();
+        baos.close();
+        ready = true;
+    }
+
+    @Override
+    public long getSize(){
+        return data.length;
+    }
+
+    @Override
+    public InputStream getInputStream() throws IOException {
+        return new ByteArrayInputStream(data);
+    }
+
+    @Override
+    public boolean isValid() {
+        return ready;
+    }
+}

src/main/java/org/leolo/map/osm/extract/util/InputUtil.java

@@ -0,0 +1,27 @@
+package org.leolo.map.osm.extract.util;
+
+import org.leolo.map.osm.extract.model.InputType;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Locale;
+
+public class InputUtil {
+    public static InputType identifyInputType(String str){
+        try {
+            URL url = new URL(str);
+        } catch (Exception e) {
+            return InputType.FILE;
+        }
+        return InputType.URL;
+    }
+
+    public static InputFile getInputFile(String path) throws IOException {
+        if(identifyInputType(path)==InputType.FILE){
+            return new FileInputFile(path);
+        }else{
+            return new URLInputFile(path);
+        }
+    }
+}

src/test/java/MainTest.java

@@ -1,14 +0,0 @@
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.ArrayList;
-
-import static org.junit.jupiter.api.Assertions.*;
-
-public class MainTest {
-
-    @Test void test1(){
-
-    }
-
-}

src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java

@@ -0,0 +1,50 @@
+package org.leolo.map.osm.extract.test;
+
+import org.junit.jupiter.api.Test;
+import org.leolo.map.osm.extract.model.InputType;
+import org.leolo.map.osm.extract.util.InputUtil;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class InputTypeTest {
+
+    @Test void testFilePath(){
+        assertEquals(InputUtil.identifyInputType("abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("../abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("./abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("..\\abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("/tmp/abc.xml"), InputType.FILE);
+    }
+
+    @Test void testUPCPath(){
+        assertEquals(InputUtil.identifyInputType("\\\\somecomputer\\somefolder\\abc.xml"), InputType.FILE);
+    }
+
+    @Test void testURL(){
+        assertEquals(InputUtil.identifyInputType("http://www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521/abc.xml"), InputType.URL);
+    }
+
+}

src/test/java/org/leolo/map/osm/extract/test/MainTest.java

@@ -0,0 +1,23 @@
+package org.leolo.map.osm.extract.test;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.leolo.map.osm.extract.util.InputFile;
+import org.leolo.map.osm.extract.util.InputUtil;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class MainTest {
+
+    @Test void test1(){
+        System.out.println("Test executed.");
+    }
+
+    @Test void downloadFile() throws IOException {
+        InputFile inputFile = InputUtil.getInputFile("https://www.leolo.pro/sample");
+        assertEquals(inputFile.length(), 8192);
+    }
+}