From 5f9302ab14ab5406ca60c160274da41917380eeb Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Tue, 20 Sep 2022 08:06:50 +0100 Subject: [PATCH 1/6] Will perform search before expanding relations --- examples/simple-action.xml | 21 +++++++++++++++++ pom.xml | 17 ++++++++++++++ .../org/leolo/map/osm/extract/ExtractElement.java | 27 ++++++++++++++++------ 3 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 examples/simple-action.xml diff --git a/examples/simple-action.xml b/examples/simple-action.xml new file mode 100644 index 0000000..fafc8f5 --- /dev/null +++ b/examples/simple-action.xml @@ -0,0 +1,21 @@ + + + + + en + de + es + fr + ja + + + short_name + website + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index fb9ada4..b58bf19 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,23 @@ junit-jupiter-api 5.9.0 + + de.topobyte + osm4j-pbf + 0.1.0 + + + + topobyte + topobyte + https://mvn.topobyte.de + + + Slimjars + Slimjars + https://mvn.slimjars.com + + \ No newline at end of file diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index c69edd1..63e2ae0 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -64,7 +64,11 @@ public class ExtractElement { private Map relations = new Hashtable<>(); private Map ways = new Hashtable<>(); private Map nodes = new Hashtable<>(); - private Set strings = new HashSet<>(); + + private HashSet pendingRelations = new HashSet<>(); + private HashSet pendingWays = new HashSet<>(); + private HashSet pendingNodes = new HashSet<>(); + private CliOpt cliOpt = new CliOpt(); @@ -146,20 +150,29 @@ public class ExtractElement { System.exit(3); return; } - expandRelations(dbFile, af, af.getRelation()); - processWay(dbFile, af); - processNode(dbFile, af); + doSearch(dbFile, af); + expandRelations(dbFile); + processWay(dbFile); + processNode(dbFile); + } + + /** + * Perform search as defined in the action file, and add them to relevant pending items + * @param dbFile The database file going to be searched + * @param af The file which defines the action to be performed + */ + private void doSearch(File dbFile, ActionFile af) { } - private void processNode(File dbFile, ActionFile af) { + private void processNode(File dbFile) { } - private void processWay(File dbFile, ActionFile af) { + private void processWay(File dbFile) { } - private void expandRelations(File dbFile, ActionFile af, Collection relations) { + private void expandRelations(File dbFile) { } -- 1.8.3.1 From c00f6ef0fb32b5216e69208858c9b51121b3eba8 Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Tue, 20 Sep 2022 08:09:11 +0100 Subject: [PATCH 2/6] Adding checks to see can a step to be skipped --- src/main/java/org/leolo/map/osm/extract/ExtractElement.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index 63e2ae0..81a8589 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -162,18 +162,22 @@ public class ExtractElement { * @param af The file which defines the action to be performed */ private void doSearch(File dbFile, ActionFile af) { + //Placeholder } private void processNode(File dbFile) { - + if(pendingNodes.isEmpty()) + return; } private void processWay(File dbFile) { - + if(pendingWays.isEmpty()) + return; } private void expandRelations(File dbFile) { - + if(pendingRelations.isEmpty()) + return; } private void printHelp(int returnValue){ -- 1.8.3.1 From 5bb3fc51faf3835693d2ebf28abbc2351210362c Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Tue, 20 Sep 2022 18:09:02 +0100 Subject: [PATCH 3/6] Provide a better error message --- src/main/java/org/leolo/map/osm/extract/ExtractElement.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index 81a8589..701dbab 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -115,7 +115,7 @@ public class ExtractElement { PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath)); File dbFile = new File(dbFilePath); if(!dbFile.exists() || !dbFile.canRead()){ - log.atError().log("Unable to read db file {}", dbFilePath); + log.atError().log("Unable to read db file {}, file cannot be read or does not exists", dbFilePath); System.exit(2); } File actionFile = new File(actionFilePath); -- 1.8.3.1 From dd385e3b3b70bef693bfa8d30848d07794b22991 Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Tue, 20 Sep 2022 20:13:41 +0100 Subject: [PATCH 4/6] Added util function to check is given string a file or url --- pom.xml | 3 +- .../org/leolo/map/osm/extract/model/InputType.java | 6 +++ .../org/leolo/map/osm/extract/util/InputUtil.java | 18 ++++++++ .../leolo/map/osm/extract/test/InputTypeTest.java | 50 ++++++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/leolo/map/osm/extract/model/InputType.java create mode 100644 src/main/java/org/leolo/map/osm/extract/util/InputUtil.java create mode 100644 src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java diff --git a/pom.xml b/pom.xml index d5622d3..a1d9d3b 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,8 @@ 3.0.0-M4 - org.leolo.map.osm.extract.test.MainTest + org.leolo.map.osm.extract.test.** + diff --git a/src/main/java/org/leolo/map/osm/extract/model/InputType.java b/src/main/java/org/leolo/map/osm/extract/model/InputType.java new file mode 100644 index 0000000..bb3e99d --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/InputType.java @@ -0,0 +1,6 @@ +package org.leolo.map.osm.extract.model; + +public enum InputType { + FILE, + URL; +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java new file mode 100644 index 0000000..9092b4f --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java @@ -0,0 +1,18 @@ +package org.leolo.map.osm.extract.util; + +import org.leolo.map.osm.extract.model.InputType; + +import java.net.MalformedURLException; +import java.net.URL; +import java.util.Locale; + +public class InputUtil { + public static InputType identifyInputType(String str){ + try { + URL url = new URL(str); + } catch (Exception e) { + return InputType.FILE; + } + return InputType.URL; + } +} diff --git a/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java b/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java new file mode 100644 index 0000000..e451e5f --- /dev/null +++ b/src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java @@ -0,0 +1,50 @@ +package org.leolo.map.osm.extract.test; + +import org.junit.jupiter.api.Test; +import org.leolo.map.osm.extract.model.InputType; +import org.leolo.map.osm.extract.util.InputUtil; + +import static org.junit.jupiter.api.Assertions.*; + +public class InputTypeTest { + + @Test void testFilePath(){ + assertEquals(InputUtil.identifyInputType("abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("../abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("./abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("..\\abc.xml"), InputType.FILE); + assertEquals(InputUtil.identifyInputType("/tmp/abc.xml"), InputType.FILE); + } + + @Test void testUPCPath(){ + assertEquals(InputUtil.identifyInputType("\\\\somecomputer\\somefolder\\abc.xml"), InputType.FILE); + } + + @Test void testURL(){ + assertEquals(InputUtil.identifyInputType("http://www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521/abc.xml"), InputType.URL); + assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521/abc.xml"), InputType.URL); + } + +} -- 1.8.3.1 From f26a861d34c72a0b8d1b395b066bc419ab542fd7 Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Tue, 20 Sep 2022 21:02:46 +0100 Subject: [PATCH 5/6] Handle having URL as db file/action file path --- .../org/leolo/map/osm/extract/ExtractElement.java | 47 +++++++---- .../leolo/map/osm/extract/model/ActionFile.java | 7 +- .../org/leolo/map/osm/extract/util/InputFile.java | 92 ++++++++++++++++++++++ .../org/leolo/map/osm/extract/util/InputUtil.java | 9 +++ 4 files changed, 136 insertions(+), 19 deletions(-) create mode 100644 src/main/java/org/leolo/map/osm/extract/util/InputFile.java diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index 701dbab..4ad5749 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -7,6 +7,8 @@ import org.apache.commons.cli.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.leolo.map.osm.extract.model.*; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; import org.xml.sax.SAXException; import javax.xml.parsers.ParserConfigurationException; @@ -58,6 +60,22 @@ public class ExtractElement { .required(true) .desc("output format") .build(); + + Option httpUserName = Option.builder() + .longOpt("http-user") + .argName("user name") + .hasArg(true) + .optionalArg(true) + .desc("User name sent during HTTP request") + .build(); + + Option httpPassword = Option.builder() + .longOpt("http-password") + .argName("passsword") + .hasArg(true) + .optionalArg(true) + .desc("Password sent during HTTP request") + .build(); } private Map allEntry = new Hashtable<>(); @@ -83,6 +101,8 @@ public class ExtractElement { String outputFilePath = null; String actionFilePath; String format; + InputFile dbFile; + InputFile actionFile; try { CommandLine cmd = parser.parse(getOptions(), args); if(args.length == 0 || cmd.hasOption(cliOpt.help)){ @@ -113,16 +133,8 @@ public class ExtractElement { System.exit(2); } PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath)); - File dbFile = new File(dbFilePath); - if(!dbFile.exists() || !dbFile.canRead()){ - log.atError().log("Unable to read db file {}, file cannot be read or does not exists", dbFilePath); - System.exit(2); - } - File actionFile = new File(actionFilePath); - if(!actionFile.exists() || !actionFile.canRead()){ - log.atError().log("Unable to read action file {}", actionFilePath); - System.exit(2); - } + dbFile = InputUtil.getInputFile(dbFilePath); + actionFile = InputUtil.getInputFile(actionFilePath); log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000); log.atInfo().log("Action file size {}kB", actionFile.length()/1_000); doExtract(dbFile, actionFile, outputStream); @@ -134,13 +146,13 @@ public class ExtractElement { System.err.println("Fatal error - unable to parse command line input!"); e.printStackTrace(); System.exit(1); - } catch (FileNotFoundException e) { + } catch (IOException e) { log.atError().withThrowable(e).log("Unable to locate the specified file"); System.exit(1); } } - private void doExtract(File dbFile, File actionFile, PrintStream outputStream) { + private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) { //Step 1: parse the actionFile ActionFile af; try { @@ -161,21 +173,21 @@ public class ExtractElement { * @param dbFile The database file going to be searched * @param af The file which defines the action to be performed */ - private void doSearch(File dbFile, ActionFile af) { + private void doSearch(InputFile dbFile, ActionFile af) { //Placeholder } - private void processNode(File dbFile) { + private void processNode(InputFile dbFile) { if(pendingNodes.isEmpty()) return; } - private void processWay(File dbFile) { + private void processWay(InputFile dbFile) { if(pendingWays.isEmpty()) return; } - private void expandRelations(File dbFile) { + private void expandRelations(InputFile dbFile) { if(pendingRelations.isEmpty()) return; } @@ -196,6 +208,9 @@ public class ExtractElement { options.addOption(cliOpt.actionFile); options.addOption(cliOpt.help); options.addOption(cliOpt.outputFormat); + //Commented first, need to find some way to pass the parameters to the function +// options.addOption(cliOpt.httpUserName); +// options.addOption(cliOpt.httpPassword); } return options; } diff --git a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java index 6179d27..e5d9f34 100644 --- a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java +++ b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java @@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.model; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.leolo.map.osm.extract.util.InputFile; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.w3c.dom.Node; @@ -23,10 +24,10 @@ public class ActionFile { private Map way = new Hashtable<>(); private Map node = new Hashtable<>(); - public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException { - log.atInfo().log("Parsing action file {}", path.getAbsolutePath()); + public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException { + log.atInfo().log("Parsing action file"); DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); - Document doc = builder.parse(path); + Document doc = builder.parse(path.getInputStream()); doc.getDocumentElement().normalize(); Element rootElement = doc.getDocumentElement(); log.atDebug().log("Root element : {}", rootElement.getTagName()); diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputFile.java b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java new file mode 100644 index 0000000..9c42ab5 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/util/InputFile.java @@ -0,0 +1,92 @@ +package org.leolo.map.osm.extract.util; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.*; +import java.net.MalformedURLException; +import java.net.URL; + +public abstract class InputFile { + public abstract long getSize(); + public abstract InputStream getInputStream() throws IOException; + + public long length() { + return getSize(); + } + + public boolean isValid(){ + return false; + } +} + +class FileInputFile extends InputFile{ + + private File f; + + FileInputFile(File f){ + this.f = f; + } + + FileInputFile(String f){ + this.f = new File(f); + } + + @Override + public long getSize(){ + return f.length(); + } + + @Override + public InputStream getInputStream() throws FileNotFoundException { + return new FileInputStream(f); + } + + @Override + public boolean isValid(){ + return f.exists() && f.canRead(); + } +} + +class URLInputFile extends InputFile{ + + private Logger log = LogManager.getLogger(); + private boolean ready = false; + + private byte[] data; + + public static final int READ_BLOCK_SIZE = 4096; + + URLInputFile(String url) throws MalformedURLException, IOException { + this(new URL(url)); + } + URLInputFile(URL url) throws IOException{ + log.atInfo().log("Connecting to {}", url); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + InputStream is = url.openStream(); + byte [] buffer = new byte[READ_BLOCK_SIZE]; + int readSize; + while((readSize = is.read(buffer))>0){ + baos.write(buffer, 0, readSize); + } + data = baos.toByteArray(); + is.close(); + baos.close(); + ready = true; + } + + @Override + public long getSize(){ + return data.length; + } + + @Override + public InputStream getInputStream() throws IOException { + return new ByteArrayInputStream(data); + } + + @Override + public boolean isValid() { + return ready; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java index 9092b4f..8f19f7c 100644 --- a/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java +++ b/src/main/java/org/leolo/map/osm/extract/util/InputUtil.java @@ -2,6 +2,7 @@ package org.leolo.map.osm.extract.util; import org.leolo.map.osm.extract.model.InputType; +import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; import java.util.Locale; @@ -15,4 +16,12 @@ public class InputUtil { } return InputType.URL; } + + public static InputFile getInputFile(String path) throws IOException { + if(identifyInputType(path)==InputType.FILE){ + return new FileInputFile(path); + }else{ + return new URLInputFile(path); + } + } } -- 1.8.3.1 From a96cc30a25dad20e637bb0ba67920a730de9ed0c Mon Sep 17 00:00:00 2001 From: LO Kam Tao Leo Date: Tue, 20 Sep 2022 21:07:20 +0100 Subject: [PATCH 6/6] Added unit test --- src/test/java/org/leolo/map/osm/extract/test/MainTest.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/test/java/org/leolo/map/osm/extract/test/MainTest.java b/src/test/java/org/leolo/map/osm/extract/test/MainTest.java index 48d3870..79c71fa 100644 --- a/src/test/java/org/leolo/map/osm/extract/test/MainTest.java +++ b/src/test/java/org/leolo/map/osm/extract/test/MainTest.java @@ -2,7 +2,10 @@ package org.leolo.map.osm.extract.test; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.leolo.map.osm.extract.util.InputFile; +import org.leolo.map.osm.extract.util.InputUtil; +import java.io.IOException; import java.util.ArrayList; import static org.junit.jupiter.api.Assertions.*; @@ -13,4 +16,8 @@ public class MainTest { System.out.println("Test executed."); } + @Test void downloadFile() throws IOException { + InputFile inputFile = InputUtil.getInputFile("https://www.leolo.pro/sample"); + assertEquals(inputFile.length(), 8192); + } } -- 1.8.3.1