diff --git a/examples/simple-action.xml b/examples/simple-action.xml new file mode 100644 index 0000000..fafc8f5 --- /dev/null +++ b/examples/simple-action.xml @@ -0,0 +1,21 @@ + + + + + en + de + es + fr + ja + + + short_name + website + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index a9fd672..a1d9d3b 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,8 @@ 3.0.0-M4 - org.leolo.map.osm.extract.test.MainTest + org.leolo.map.osm.extract.test.** + @@ -64,6 +65,23 @@ junit-jupiter-api 5.9.0 + + de.topobyte + osm4j-pbf + 0.1.0 + + + + topobyte + topobyte + https://mvn.topobyte.de + + + Slimjars + Slimjars + https://mvn.slimjars.com + + \ No newline at end of file diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index c69edd1..4ad5749 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -7,6 +7,8 @@ import org.apache.commons.cli.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.leolo.map.osm.extract.model.*; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; @@ -58,13 +60,33 @@ public class ExtractElement { .required(true) .desc("output format") .build(); + + Option httpUserName = Option.builder() + .longOpt("http-user") + .argName("user name") + .hasArg(true) + .optionalArg(true) + .desc("User name sent during HTTP request") + .build(); + + Option httpPassword = Option.builder() + .longOpt("http-password") + .argName("passsword") + .hasArg(true) + .optionalArg(true) + .desc("Password sent during HTTP request") + .build(); } private Map allEntry = new Hashtable<>(); private Map relations = new Hashtable<>(); private Map ways = new Hashtable<>(); private Map nodes = new Hashtable<>(); - private Set strings = new HashSet<>(); + + private HashSet pendingRelations = new HashSet<>(); + private HashSet pendingWays = new HashSet<>(); + private HashSet pendingNodes = new HashSet<>(); + private CliOpt cliOpt = new CliOpt(); @@ -79,6 +101,8 @@ public class ExtractElement { String outputFilePath = null; String actionFilePath; String format; + InputFile dbFile; + InputFile actionFile; try { CommandLine cmd = parser.parse(getOptions(), args); if(args.length == 0 || cmd.hasOption(cliOpt.help)){ @@ -109,16 +133,8 @@ public class ExtractElement { System.exit(2); } PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath)); - File dbFile = new File(dbFilePath); - if(!dbFile.exists() || !dbFile.canRead()){ - log.atError().log("Unable to read db file {}", dbFilePath); - System.exit(2); - } - File actionFile = new File(actionFilePath); - if(!actionFile.exists() || !actionFile.canRead()){ - log.atError().log("Unable to read action file {}", actionFilePath); - System.exit(2); - } + dbFile = InputUtil.getInputFile(dbFilePath); + actionFile = InputUtil.getInputFile(actionFilePath); log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000); log.atInfo().log("Action file size {}kB", actionFile.length()/1_000); doExtract(dbFile, actionFile, outputStream); @@ -130,13 +146,13 @@ public class ExtractElement { System.err.println("Fatal error - unable to parse command line input!"); e.printStackTrace(); System.exit(1); - } catch (FileNotFoundException e) { + } catch (IOException e) { log.atError().withThrowable(e).log("Unable to locate the specified file"); System.exit(1); } } - private void doExtract(File dbFile, File actionFile, PrintStream outputStream) { + private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) { //Step 1: parse the actionFile ActionFile af; try { @@ -146,21 +162,34 @@ public class ExtractElement { System.exit(3); return; } - expandRelations(dbFile, af, af.getRelation()); - processWay(dbFile, af); - processNode(dbFile, af); + doSearch(dbFile, af); + expandRelations(dbFile); + processWay(dbFile); + processNode(dbFile); } - private void processNode(File dbFile, ActionFile af) { - + /** + * Perform search as defined in the action file, and add them to relevant pending items + * @param dbFile The database file going to be searched + * @param af The file which defines the action to be performed + */ + private void doSearch(InputFile dbFile, ActionFile af) { + //Placeholder } - private void processWay(File dbFile, ActionFile af) { - + private void processNode(InputFile dbFile) { + if(pendingNodes.isEmpty()) + return; } - private void expandRelations(File dbFile, ActionFile af, Collection relations) { + private void processWay(InputFile dbFile) { + if(pendingWays.isEmpty()) + return; + } + private void expandRelations(InputFile dbFile) { + if(pendingRelations.isEmpty()) + return; } private void printHelp(int returnValue){ @@ -179,6 +208,9 @@ public class ExtractElement { options.addOption(cliOpt.actionFile); options.addOption(cliOpt.help); options.addOption(cliOpt.outputFormat); + //Commented first, need to find some way to pass the parameters to the function +// options.addOption(cliOpt.httpUserName); +// options.addOption(cliOpt.httpPassword); } return options; } diff --git a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java index 6179d27..e5d9f34 100644 --- a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java +++ b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java @@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.leolo.map.osm.extract.util.InputFile; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -23,10 +24,10 @@ public class ActionFile { private Map way = new Hashtable<>(); private Map node = new Hashtable<>(); - public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException { - log.atInfo().log("Parsing action file {}", path.getAbsolutePath()); + public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException { + log.atInfo().log("Parsing action file"); DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Document doc = builder.parse(path); + Document doc = builder.parse(path.getInputStream()); doc.getDocumentElement().normalize(); Element rootElement = doc.getDocumentElement(); log.atDebug().log("Root element : {}", rootElement.getTagName()); diff --git a/src/main/java/org/leolo/map/osm/extract/model/InputType.java b/src/main/java/org/leolo/map/osm/extract/model/InputType.java new file mode 100644 index 0000000..bb3e99d --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/InputType.java @@ -0,0 +1,6 @@ +package org.leolo.map.osm.extract.model; + +public enum InputType { + FILE, + URL; +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputFile.java b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java new file mode 100644 index 0000000..9c42ab5 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java @@ -0,0 +1,92 @@ +package org.leolo.map.osm.extract.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URL; + +public abstract class InputFile { + public abstract long getSize(); + public abstract InputStream getInputStream() throws IOException; + + public long length() { + return getSize(); + } + + public boolean isValid(){ + return false; + } +} + +class FileInputFile extends InputFile{ + + private File f; + + FileInputFile(File f){ + this.f = f; + } + + FileInputFile(String f){ + this.f = new File(f); + } + + @Override + public long getSize(){ + return f.length(); + } + + @Override + public InputStream getInputStream() throws FileNotFoundException { + return new FileInputStream(f); + } + + @Override + public boolean isValid(){ + return f.exists() && f.canRead(); + } +} + +class URLInputFile extends InputFile{ + + private Logger log = LogManager.getLogger(); + private boolean ready = false; + + private byte[] data; + + public static final int READ_BLOCK_SIZE = 4096; + + URLInputFile(String url) throws MalformedURLException, IOException { + this(new URL(url)); + } + URLInputFile(URL url) throws IOException{ + log.atInfo().log("Connecting to {}", url); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InputStream is = url.openStream(); + byte [] buffer = new byte[READ_BLOCK_SIZE]; + int readSize; + while((readSize = is.read(buffer))>0){ + baos.write(buffer, 0, readSize); + } + data = baos.toByteArray(); + is.close(); + baos.close(); + ready = true; + } + + @Override + public long getSize(){ + return data.length; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(data); + } + + @Override + public boolean isValid() { + return ready; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java new file mode 100644 index 0000000..8f19f7c --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java @@ -0,0 +1,27 @@ +package org.leolo.map.osm.extract.util; + +import org.leolo.map.osm.extract.model.InputType; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Locale; + +public class InputUtil { + public static InputType identifyInputType(String str){ + try { + URL url = new URL(str); + } catch (Exception e) { + return InputType.FILE; + } + return InputType.URL; + } + + public static InputFile getInputFile(String path) throws IOException { + if(identifyInputType(path)==InputType.FILE){ + return new FileInputFile(path); + }else{ + return new URLInputFile(path); + } + } +} diff --git a/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java b/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java new file mode 100644 index 0000000..e451e5f --- /dev/null +++ b/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java @@ -0,0 +1,50 @@ +package org.leolo.map.osm.extract.test; + +import org.junit.jupiter.api.Test; +import org.leolo.map.osm.extract.model.InputType; +import org.leolo.map.osm.extract.util.InputUtil; + +import static org.junit.jupiter.api.Assertions.*; + +public class InputTypeTest { + + @Test void testFilePath(){ + assertEquals(InputUtil.identifyInputType("abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("../abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("./abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("..\\abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("/tmp/abc.xml"), InputType.FILE); + } + + @Test void testUPCPath(){ + assertEquals(InputUtil.identifyInputType("\\\\somecomputer\\somefolder\\abc.xml"), InputType.FILE); + } + + @Test void testURL(){ + assertEquals(InputUtil.identifyInputType("http://www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521/abc.xml"), InputType.URL); + } + +} diff --git a/src/test/java/org/leolo/map/osm/extract/test/MainTest.java b/src/test/java/org/leolo/map/osm/extract/test/MainTest.java index 48d3870..79c71fa 100644 --- a/src/test/java/org/leolo/map/osm/extract/test/MainTest.java +++ b/src/test/java/org/leolo/map/osm/extract/test/MainTest.java @@ -2,7 +2,10 @@ package org.leolo.map.osm.extract.test; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; +import java.io.IOException; import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.*; @@ -13,4 +16,8 @@ public class MainTest { System.out.println("Test executed."); } + @Test void downloadFile() throws IOException { + InputFile inputFile = InputUtil.getInputFile("https://www.leolo.pro/sample"); + assertEquals(inputFile.length(), 8192); + } }