diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index 701dbab..4ad5749 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -7,6 +7,8 @@ import org.apache.commons.cli.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.leolo.map.osm.extract.model.*; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; @@ -58,6 +60,22 @@ public class ExtractElement { .required(true) .desc("output format") .build(); + + Option httpUserName = Option.builder() + .longOpt("http-user") + .argName("user name") + .hasArg(true) + .optionalArg(true) + .desc("User name sent during HTTP request") + .build(); + + Option httpPassword = Option.builder() + .longOpt("http-password") + .argName("passsword") + .hasArg(true) + .optionalArg(true) + .desc("Password sent during HTTP request") + .build(); } private Map allEntry = new Hashtable<>(); @@ -83,6 +101,8 @@ public class ExtractElement { String outputFilePath = null; String actionFilePath; String format; + InputFile dbFile; + InputFile actionFile; try { CommandLine cmd = parser.parse(getOptions(), args); if(args.length == 0 || cmd.hasOption(cliOpt.help)){ @@ -113,16 +133,8 @@ public class ExtractElement { System.exit(2); } PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath)); - File dbFile = new File(dbFilePath); - if(!dbFile.exists() || !dbFile.canRead()){ - log.atError().log("Unable to read db file {}, file cannot be read or does not exists", dbFilePath); - System.exit(2); - } - File actionFile = new File(actionFilePath); - if(!actionFile.exists() || !actionFile.canRead()){ - log.atError().log("Unable to read action file {}", actionFilePath); - System.exit(2); - } + dbFile = InputUtil.getInputFile(dbFilePath); + actionFile = InputUtil.getInputFile(actionFilePath); log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000); log.atInfo().log("Action file size {}kB", actionFile.length()/1_000); doExtract(dbFile, actionFile, outputStream); @@ -134,13 +146,13 @@ public class ExtractElement { System.err.println("Fatal error - unable to parse command line input!"); e.printStackTrace(); System.exit(1); - } catch (FileNotFoundException e) { + } catch (IOException e) { log.atError().withThrowable(e).log("Unable to locate the specified file"); System.exit(1); } } - private void doExtract(File dbFile, File actionFile, PrintStream outputStream) { + private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) { //Step 1: parse the actionFile ActionFile af; try { @@ -161,21 +173,21 @@ public class ExtractElement { * @param dbFile The database file going to be searched * @param af The file which defines the action to be performed */ - private void doSearch(File dbFile, ActionFile af) { + private void doSearch(InputFile dbFile, ActionFile af) { //Placeholder } - private void processNode(File dbFile) { + private void processNode(InputFile dbFile) { if(pendingNodes.isEmpty()) return; } - private void processWay(File dbFile) { + private void processWay(InputFile dbFile) { if(pendingWays.isEmpty()) return; } - private void expandRelations(File dbFile) { + private void expandRelations(InputFile dbFile) { if(pendingRelations.isEmpty()) return; } @@ -196,6 +208,9 @@ public class ExtractElement { options.addOption(cliOpt.actionFile); options.addOption(cliOpt.help); options.addOption(cliOpt.outputFormat); + //Commented first, need to find some way to pass the parameters to the function +// options.addOption(cliOpt.httpUserName); +// options.addOption(cliOpt.httpPassword); } return options; } diff --git a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java index 6179d27..e5d9f34 100644 --- a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java +++ b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java @@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.leolo.map.osm.extract.util.InputFile; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -23,10 +24,10 @@ public class ActionFile { private Map way = new Hashtable<>(); private Map node = new Hashtable<>(); - public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException { - log.atInfo().log("Parsing action file {}", path.getAbsolutePath()); + public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException { + log.atInfo().log("Parsing action file"); DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Document doc = builder.parse(path); + Document doc = builder.parse(path.getInputStream()); doc.getDocumentElement().normalize(); Element rootElement = doc.getDocumentElement(); log.atDebug().log("Root element : {}", rootElement.getTagName()); diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputFile.java b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java new file mode 100644 index 0000000..9c42ab5 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java @@ -0,0 +1,92 @@ +package org.leolo.map.osm.extract.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URL; + +public abstract class InputFile { + public abstract long getSize(); + public abstract InputStream getInputStream() throws IOException; + + public long length() { + return getSize(); + } + + public boolean isValid(){ + return false; + } +} + +class FileInputFile extends InputFile{ + + private File f; + + FileInputFile(File f){ + this.f = f; + } + + FileInputFile(String f){ + this.f = new File(f); + } + + @Override + public long getSize(){ + return f.length(); + } + + @Override + public InputStream getInputStream() throws FileNotFoundException { + return new FileInputStream(f); + } + + @Override + public boolean isValid(){ + return f.exists() && f.canRead(); + } +} + +class URLInputFile extends InputFile{ + + private Logger log = LogManager.getLogger(); + private boolean ready = false; + + private byte[] data; + + public static final int READ_BLOCK_SIZE = 4096; + + URLInputFile(String url) throws MalformedURLException, IOException { + this(new URL(url)); + } + URLInputFile(URL url) throws IOException{ + log.atInfo().log("Connecting to {}", url); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InputStream is = url.openStream(); + byte [] buffer = new byte[READ_BLOCK_SIZE]; + int readSize; + while((readSize = is.read(buffer))>0){ + baos.write(buffer, 0, readSize); + } + data = baos.toByteArray(); + is.close(); + baos.close(); + ready = true; + } + + @Override + public long getSize(){ + return data.length; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(data); + } + + @Override + public boolean isValid() { + return ready; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java index 9092b4f..8f19f7c 100644 --- a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java +++ b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java @@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.util; import org.leolo.map.osm.extract.model.InputType; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.Locale; @@ -15,4 +16,12 @@ public class InputUtil { } return InputType.URL; } + + public static InputFile getInputFile(String path) throws IOException { + if(identifyInputType(path)==InputType.FILE){ + return new FileInputFile(path); + }else{ + return new URLInputFile(path); + } + } }