Handle having URL as db file/action file path

src/main/java/org/leolo/map/osm/extract/ExtractElement.java

@@ -7,6 +7,8 @@ import org.apache.commons.cli.*;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.leolo.map.osm.extract.model.*;
+import org.leolo.map.osm.extract.util.InputFile;
+import org.leolo.map.osm.extract.util.InputUtil;
 import org.xml.sax.SAXException;
 
 import javax.xml.parsers.ParserConfigurationException;
@@ -58,6 +60,22 @@ public class ExtractElement {
                 .required(true)
                 .desc("output format")
                 .build();
+
+        Option httpUserName = Option.builder()
+                .longOpt("http-user")
+                .argName("user name")
+                .hasArg(true)
+                .optionalArg(true)
+                .desc("User name sent during HTTP request")
+                .build();
+
+        Option httpPassword = Option.builder()
+                .longOpt("http-password")
+                .argName("passsword")
+                .hasArg(true)
+                .optionalArg(true)
+                .desc("Password sent during HTTP request")
+                .build();
     }
 
     private Map<Long, MapEntry> allEntry = new Hashtable<>();
@@ -83,6 +101,8 @@ public class ExtractElement {
         String outputFilePath = null;
         String actionFilePath;
         String format;
+        InputFile dbFile;
+        InputFile actionFile;
         try {
             CommandLine cmd = parser.parse(getOptions(), args);
             if(args.length == 0 || cmd.hasOption(cliOpt.help)){
@@ -113,16 +133,8 @@ public class ExtractElement {
                 System.exit(2);
             }
             PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath));
-            File dbFile = new File(dbFilePath);
+            dbFile = InputUtil.getInputFile(dbFilePath);
-            if(!dbFile.exists() || !dbFile.canRead()){
+            actionFile = InputUtil.getInputFile(actionFilePath);
-                log.atError().log("Unable to read db file {}, file cannot be read or does not exists", dbFilePath);
-                System.exit(2);
-            }
-            File actionFile = new File(actionFilePath);
-            if(!actionFile.exists() || !actionFile.canRead()){
-                log.atError().log("Unable to read action file {}", actionFilePath);
-                System.exit(2);
-            }
             log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000);
             log.atInfo().log("Action file size {}kB", actionFile.length()/1_000);
             doExtract(dbFile, actionFile, outputStream);
@@ -134,13 +146,13 @@ public class ExtractElement {
             System.err.println("Fatal error - unable to parse command line input!");
             e.printStackTrace();
             System.exit(1);
-        } catch (FileNotFoundException e) {
+        } catch (IOException e) {
             log.atError().withThrowable(e).log("Unable to locate the specified file");
             System.exit(1);
         }
     }
 
-    private void doExtract(File dbFile, File actionFile, PrintStream outputStream) {
+    private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) {
         //Step 1: parse the actionFile
         ActionFile af;
         try {
@@ -161,21 +173,21 @@ public class ExtractElement {
      * @param dbFile The database file going to be searched
      * @param af The file which defines the action to be performed
      */
-    private void doSearch(File dbFile, ActionFile af) {
+    private void doSearch(InputFile dbFile, ActionFile af) {
         //Placeholder
     }
 
-    private void processNode(File dbFile) {
+    private void processNode(InputFile dbFile) {
         if(pendingNodes.isEmpty())
             return;
     }
 
-    private void processWay(File dbFile) {
+    private void processWay(InputFile dbFile) {
         if(pendingWays.isEmpty())
             return;
     }
 
-    private void expandRelations(File dbFile) {
+    private void expandRelations(InputFile dbFile) {
         if(pendingRelations.isEmpty())
             return;
     }
@@ -196,6 +208,9 @@ public class ExtractElement {
             options.addOption(cliOpt.actionFile);
             options.addOption(cliOpt.help);
             options.addOption(cliOpt.outputFormat);
+            //Commented first, need to find some way to pass the parameters to the function
+//            options.addOption(cliOpt.httpUserName);
+//            options.addOption(cliOpt.httpPassword);
         }
         return options;
     }

src/main/java/org/leolo/map/osm/extract/model/ActionFile.java

@@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.leolo.map.osm.extract.util.InputFile;
 import org.w3c.dom.Document;
 import org.w3c.dom.Element;
 import org.w3c.dom.Node;
@@ -23,10 +24,10 @@ public class ActionFile {
     private Map<Long,ExtractItem> way = new Hashtable<>();
     private Map<Long,ExtractItem> node = new Hashtable<>();
 
-    public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException {
+    public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException {
-        log.atInfo().log("Parsing action file {}", path.getAbsolutePath());
+        log.atInfo().log("Parsing action file");
         DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
-        Document doc = builder.parse(path);
+        Document doc = builder.parse(path.getInputStream());
         doc.getDocumentElement().normalize();
         Element rootElement = doc.getDocumentElement();
         log.atDebug().log("Root element : {}", rootElement.getTagName());

src/main/java/org/leolo/map/osm/extract/util/InputFile.java

@@ -0,0 +1,92 @@
+package org.leolo.map.osm.extract.util;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.io.*;
+import java.net.MalformedURLException;
+import java.net.URL;
+
+public abstract class InputFile {
+    public abstract long getSize();
+    public abstract InputStream getInputStream() throws IOException;
+
+    public long length() {
+        return getSize();
+    }
+
+    public boolean isValid(){
+        return false;
+    }
+}
+
+class FileInputFile extends InputFile{
+
+    private File f;
+
+    FileInputFile(File f){
+        this.f = f;
+    }
+
+    FileInputFile(String f){
+        this.f = new File(f);
+    }
+
+    @Override
+    public long getSize(){
+        return f.length();
+    }
+
+    @Override
+    public InputStream getInputStream() throws FileNotFoundException {
+        return new FileInputStream(f);
+    }
+
+    @Override
+    public boolean isValid(){
+        return f.exists() && f.canRead();
+    }
+}
+
+class URLInputFile extends InputFile{
+
+    private Logger log = LogManager.getLogger();
+    private boolean ready = false;
+
+    private byte[] data;
+
+    public static final int READ_BLOCK_SIZE = 4096;
+
+    URLInputFile(String url) throws MalformedURLException, IOException {
+        this(new URL(url));
+    }
+    URLInputFile(URL url) throws IOException{
+        log.atInfo().log("Connecting to {}", url);
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        InputStream is = url.openStream();
+        byte [] buffer = new byte[READ_BLOCK_SIZE];
+        int readSize;
+        while((readSize = is.read(buffer))>0){
+            baos.write(buffer, 0, readSize);
+        }
+        data = baos.toByteArray();
+        is.close();
+        baos.close();
+        ready = true;
+    }
+
+    @Override
+    public long getSize(){
+        return data.length;
+    }
+
+    @Override
+    public InputStream getInputStream() throws IOException {
+        return new ByteArrayInputStream(data);
+    }
+
+    @Override
+    public boolean isValid() {
+        return ready;
+    }
+}

src/main/java/org/leolo/map/osm/extract/util/InputUtil.java

@@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.util;
 
 import org.leolo.map.osm.extract.model.InputType;
 
+import java.io.IOException;
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Locale;
@@ -15,4 +16,12 @@ public class InputUtil {
         }
         return InputType.URL;
     }
+
+    public static InputFile getInputFile(String path) throws IOException {
+        if(identifyInputType(path)==InputType.FILE){
+            return new FileInputFile(path);
+        }else{
+            return new URLInputFile(path);
+        }
+    }
 }