feature/downloadDatabase (#3)

Co-authored-by: LO Kam Tao Leo <leolo@leolo.org>
Reviewed-on: #3

examples/simple-action.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<actions>
+    <properties>
+        <lang>
+            <default>en</default>
+            <other>de</other>
+            <other>es</other>
+            <other>fr</other>
+            <other>ja</other>
+        </lang>
+        <tags>
+            <display>short_name</display>
+            <display>website</display>
+        </tags>
+    </properties>
+    <extracts>
+        <relation id="9936954"/>
+        <way id="79289788"/>
+        <node id="6720935522"/>
+    </extracts>
+</actions>

pom.xml

@@ -22,7 +22,8 @@
                 <version>3.0.0-M4</version>
                 <configuration>
                     <includes>
-                        <include>org.leolo.map.osm.extract.test.MainTest</include>
+                        <include>org.leolo.map.osm.extract.test.**</include>
+
                     </includes>
                 </configuration>
             </plugin>
@@ -64,6 +65,23 @@
             <artifactId>junit-jupiter-api</artifactId>
             <version>5.9.0</version>
         </dependency>
+        <dependency>
+            <groupId>de.topobyte</groupId>
+            <artifactId>osm4j-pbf</artifactId>
+            <version>0.1.0</version>
+        </dependency>
     </dependencies>
+    <repositories>
+        <repository>
+            <id>topobyte</id>
+            <name>topobyte</name>
+            <url>https://mvn.topobyte.de</url>
+        </repository>
+        <repository>
+            <id>Slimjars</id>
+            <name>Slimjars</name>
+            <url>https://mvn.slimjars.com</url>
+        </repository>
+    </repositories>
 
 </project>

src/main/java/org/leolo/map/osm/extract/ExtractElement.java

@@ -7,6 +7,8 @@ import org.apache.commons.cli.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.leolo.map.osm.extract.model.*;
+import org.leolo.map.osm.extract.util.InputFile;
+import org.leolo.map.osm.extract.util.InputUtil;
 import org.xml.sax.SAXException;
 
 import javax.xml.parsers.ParserConfigurationException;
@@ -58,13 +60,33 @@ public class ExtractElement {
                 .required(true)
                 .desc("output format")
                 .build();
+
+        Option httpUserName = Option.builder()
+                .longOpt("http-user")
+                .argName("user name")
+                .hasArg(true)
+                .optionalArg(true)
+                .desc("User name sent during HTTP request")
+                .build();
+
+        Option httpPassword = Option.builder()
+                .longOpt("http-password")
+                .argName("passsword")
+                .hasArg(true)
+                .optionalArg(true)
+                .desc("Password sent during HTTP request")
+                .build();
     }
 
     private Map<Long, MapEntry> allEntry = new Hashtable<>();
     private Map<Long, Relation> relations = new Hashtable<>();
     private Map<Long, Way> ways = new Hashtable<>();
     private Map<Long, Node> nodes = new Hashtable<>();
-    private Set<Integer> strings = new HashSet<>();
+
+    private HashSet<Long> pendingRelations = new HashSet<>();
+    private HashSet<Long> pendingWays = new HashSet<>();
+    private HashSet<Long> pendingNodes = new HashSet<>();
+
 
 
     private CliOpt cliOpt = new CliOpt();
@@ -79,6 +101,8 @@ public class ExtractElement {
         String outputFilePath = null;
         String actionFilePath;
         String format;
+        InputFile dbFile;
+        InputFile actionFile;
         try {
             CommandLine cmd = parser.parse(getOptions(), args);
             if(args.length == 0 || cmd.hasOption(cliOpt.help)){
@@ -109,16 +133,8 @@ public class ExtractElement {
                 System.exit(2);
             }
             PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath));
-            File dbFile = new File(dbFilePath);
+            dbFile = InputUtil.getInputFile(dbFilePath);
-            if(!dbFile.exists() || !dbFile.canRead()){
+            actionFile = InputUtil.getInputFile(actionFilePath);
-                log.atError().log("Unable to read db file {}", dbFilePath);
-                System.exit(2);
-            }
-            File actionFile = new File(actionFilePath);
-            if(!actionFile.exists() || !actionFile.canRead()){
-                log.atError().log("Unable to read action file {}", actionFilePath);
-                System.exit(2);
-            }
             log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000);
             log.atInfo().log("Action file size {}kB", actionFile.length()/1_000);
             doExtract(dbFile, actionFile, outputStream);
@@ -130,13 +146,13 @@ public class ExtractElement {
             System.err.println("Fatal error - unable to parse command line input!");
             e.printStackTrace();
             System.exit(1);
-        } catch (FileNotFoundException e) {
+        } catch (IOException e) {
             log.atError().withThrowable(e).log("Unable to locate the specified file");
             System.exit(1);
         }
     }
 
-    private void doExtract(File dbFile, File actionFile, PrintStream outputStream) {
+    private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) {
         //Step 1: parse the actionFile
         ActionFile af;
         try {
@@ -146,21 +162,34 @@ public class ExtractElement {
             System.exit(3);
             return;
         }
-        expandRelations(dbFile, af, af.getRelation());
+        doSearch(dbFile, af);
-        processWay(dbFile, af);
+        expandRelations(dbFile);
-        processNode(dbFile, af);
+        processWay(dbFile);
+        processNode(dbFile);
     }
 
-    private void processNode(File dbFile, ActionFile af) {
+    /**
-
+     * Perform search as defined in the action file, and add them to relevant pending items
+     * @param dbFile The database file going to be searched
+     * @param af The file which defines the action to be performed
+     */
+    private void doSearch(InputFile dbFile, ActionFile af) {
+        //Placeholder
     }
 
-    private void processWay(File dbFile, ActionFile af) {
+    private void processNode(InputFile dbFile) {
-
+        if(pendingNodes.isEmpty())
+            return;
     }
 
-    private void expandRelations(File dbFile, ActionFile af, Collection<ExtractItem> relations) {
+    private void processWay(InputFile dbFile) {
+        if(pendingWays.isEmpty())
+            return;
+    }
 
+    private void expandRelations(InputFile dbFile) {
+        if(pendingRelations.isEmpty())
+            return;
     }
 
     private void printHelp(int returnValue){
@@ -179,6 +208,9 @@ public class ExtractElement {
             options.addOption(cliOpt.actionFile);
             options.addOption(cliOpt.help);
             options.addOption(cliOpt.outputFormat);
+            //Commented first, need to find some way to pass the parameters to the function
+//            options.addOption(cliOpt.httpUserName);
+//            options.addOption(cliOpt.httpPassword);
         }
         return options;
     }

src/main/java/org/leolo/map/osm/extract/model/ActionFile.java

@@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.leolo.map.osm.extract.util.InputFile;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -23,10 +24,10 @@ public class ActionFile {
     private Map<Long,ExtractItem> way = new Hashtable<>();
     private Map<Long,ExtractItem> node = new Hashtable<>();
 
-    public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException {
+    public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException {
-        log.atInfo().log("Parsing action file {}", path.getAbsolutePath());
+        log.atInfo().log("Parsing action file");
         DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-        Document doc = builder.parse(path);
+        Document doc = builder.parse(path.getInputStream());
         doc.getDocumentElement().normalize();
         Element rootElement = doc.getDocumentElement();
         log.atDebug().log("Root element : {}", rootElement.getTagName());

src/main/java/org/leolo/map/osm/extract/model/InputType.java

@@ -0,0 +1,6 @@
+package org.leolo.map.osm.extract.model;
+
+public enum InputType {
+    FILE,
+    URL;
+}

src/main/java/org/leolo/map/osm/extract/util/InputFile.java

@@ -0,0 +1,92 @@
+package org.leolo.map.osm.extract.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public abstract class InputFile {
+    public abstract long getSize();
+    public abstract InputStream getInputStream() throws IOException;
+
+    public long length() {
+        return getSize();
+    }
+
+    public boolean isValid(){
+        return false;
+    }
+}
+
+class FileInputFile extends InputFile{
+
+    private File f;
+
+    FileInputFile(File f){
+        this.f = f;
+    }
+
+    FileInputFile(String f){
+        this.f = new File(f);
+    }
+
+    @Override
+    public long getSize(){
+        return f.length();
+    }
+
+    @Override
+    public InputStream getInputStream() throws FileNotFoundException {
+        return new FileInputStream(f);
+    }
+
+    @Override
+    public boolean isValid(){
+        return f.exists() && f.canRead();
+    }
+}
+
+class URLInputFile extends InputFile{
+
+    private Logger log = LogManager.getLogger();
+    private boolean ready = false;
+
+    private byte[] data;
+
+    public static final int READ_BLOCK_SIZE = 4096;
+
+    URLInputFile(String url) throws MalformedURLException, IOException {
+        this(new URL(url));
+    }
+    URLInputFile(URL url) throws IOException{
+        log.atInfo().log("Connecting to {}", url);
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        InputStream is = url.openStream();
+        byte [] buffer = new byte[READ_BLOCK_SIZE];
+        int readSize;
+        while((readSize = is.read(buffer))>0){
+            baos.write(buffer, 0, readSize);
+        }
+        data = baos.toByteArray();
+        is.close();
+        baos.close();
+        ready = true;
+    }
+
+    @Override
+    public long getSize(){
+        return data.length;
+    }
+
+    @Override
+    public InputStream getInputStream() throws IOException {
+        return new ByteArrayInputStream(data);
+    }
+
+    @Override
+    public boolean isValid() {
+        return ready;
+    }
+}

src/main/java/org/leolo/map/osm/extract/util/InputUtil.java

@@ -0,0 +1,27 @@
+package org.leolo.map.osm.extract.util;
+
+import org.leolo.map.osm.extract.model.InputType;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Locale;
+
+public class InputUtil {
+    public static InputType identifyInputType(String str){
+        try {
+            URL url = new URL(str);
+        } catch (Exception e) {
+            return InputType.FILE;
+        }
+        return InputType.URL;
+    }
+
+    public static InputFile getInputFile(String path) throws IOException {
+        if(identifyInputType(path)==InputType.FILE){
+            return new FileInputFile(path);
+        }else{
+            return new URLInputFile(path);
+        }
+    }
+}

src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java

@@ -0,0 +1,50 @@
+package org.leolo.map.osm.extract.test;
+
+import org.junit.jupiter.api.Test;
+import org.leolo.map.osm.extract.model.InputType;
+import org.leolo.map.osm.extract.util.InputUtil;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+public class InputTypeTest {
+
+    @Test void testFilePath(){
+        assertEquals(InputUtil.identifyInputType("abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("../abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("./abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("..\\abc.xml"), InputType.FILE);
+        assertEquals(InputUtil.identifyInputType("/tmp/abc.xml"), InputType.FILE);
+    }
+
+    @Test void testUPCPath(){
+        assertEquals(InputUtil.identifyInputType("\\\\somecomputer\\somefolder\\abc.xml"), InputType.FILE);
+    }
+
+    @Test void testURL(){
+        assertEquals(InputUtil.identifyInputType("http://www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521/abc.xml"), InputType.URL);
+        assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521/abc.xml"), InputType.URL);
+    }
+
+}

src/test/java/org/leolo/map/osm/extract/test/MainTest.java

@@ -2,7 +2,10 @@ package org.leolo.map.osm.extract.test;
 
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
+import org.leolo.map.osm.extract.util.InputFile;
+import org.leolo.map.osm.extract.util.InputUtil;
 
+import java.io.IOException;
 import java.util.ArrayList;
 
 import static org.junit.jupiter.api.Assertions.*;
@@ -13,4 +16,8 @@ public class MainTest {
         System.out.println("Test executed.");
     }
 
+    @Test void downloadFile() throws IOException {
+        InputFile inputFile = InputUtil.getInputFile("https://www.leolo.pro/sample");
+        assertEquals(inputFile.length(), 8192);
+    }
 }