Compare commits

...

14 Commits

  1. 2
      .gitignore
  2. 174
      .idea/workspace.xml
  3. 2
      Jenkinsfile
  4. 21
      examples/simple-action.xml
  5. 43
      pom.xml
  6. 286
      src/main/java/org/leolo/map/osm/extract/ExtractElement.java
  7. 40
      src/main/java/org/leolo/map/osm/extract/ExtractResult.java
  8. 8
      src/main/java/org/leolo/map/osm/extract/OutputFormat.java
  9. 12
      src/main/java/org/leolo/map/osm/extract/SearchProvider.java
  10. 112
      src/main/java/org/leolo/map/osm/extract/SearchProviderManager.java
  11. 82
      src/main/java/org/leolo/map/osm/extract/format/KMLFormatter.java
  12. 4
      src/main/java/org/leolo/map/osm/extract/format/PlainFormatter.java
  13. 24
      src/main/java/org/leolo/map/osm/extract/format/SimpleStringSearchProvider.java
  14. 63
      src/main/java/org/leolo/map/osm/extract/model/ActionFile.java
  15. 3
      src/main/java/org/leolo/map/osm/extract/model/ExtractItem.java
  16. 6
      src/main/java/org/leolo/map/osm/extract/model/InputType.java
  17. 4
      src/main/java/org/leolo/map/osm/extract/model/MapEntry.java
  18. 16
      src/main/java/org/leolo/map/osm/extract/model/Node.java
  19. 9
      src/main/java/org/leolo/map/osm/extract/model/Relation.java
  20. 68
      src/main/java/org/leolo/map/osm/extract/model/SearchItem.java
  21. 92
      src/main/java/org/leolo/map/osm/extract/util/InputFile.java
  22. 27
      src/main/java/org/leolo/map/osm/extract/util/InputUtil.java
  23. 14
      src/test/java/MainTest.java
  24. 50
      src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java
  25. 23
      src/test/java/org/leolo/map/osm/extract/test/MainTest.java

2
.gitignore vendored

@ -38,3 +38,5 @@ build/
.DS_Store
ri_log
action.xml
.idea/
output/

174
.idea/workspace.xml

@ -1,174 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="AutoImportSettings">
<option name="autoReloadType" value="SELECTIVE" />
</component>
<component name="ChangeListManager">
<list default="true" id="aada2b3d-ba01-45e9-89c7-a596447b137a" name="Changes" comment="">
<change afterPath="$PROJECT_DIR$/src/test/java/MainTest.java" afterDir="false" />
<change beforePath="$PROJECT_DIR$/.idea/workspace.xml" beforeDir="false" afterPath="$PROJECT_DIR$/.idea/workspace.xml" afterDir="false" />
<change beforePath="$PROJECT_DIR$/pom.xml" beforeDir="false" afterPath="$PROJECT_DIR$/pom.xml" afterDir="false" />
</list>
<option name="SHOW_DIALOG" value="false" />
<option name="HIGHLIGHT_CONFLICTS" value="true" />
<option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
<option name="LAST_RESOLUTION" value="IGNORE" />
</component>
<component name="FileTemplateManagerImpl">
<option name="RECENT_TEMPLATES">
<list>
<option value="AnnotationType" />
<option value="Interface" />
<option value="Class" />
</list>
</option>
</component>
<component name="Git.Settings">
<option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
</component>
<component name="MarkdownSettingsMigration">
<option name="stateVersion" value="1" />
</component>
<component name="ProjectId" id="2Eykj2SggZ4KtR6OJyjcndHtTWt" />
<component name="ProjectLevelVcsManager" settingsEditedManually="true" />
<component name="ProjectViewState">
<option name="hideEmptyMiddlePackages" value="true" />
<option name="showLibraryContents" value="true" />
</component>
<component name="PropertiesComponent"><![CDATA[{
"keyToString": {
"RunOnceActivity.OpenProjectViewOnStart": "true",
"RunOnceActivity.ShowReadmeOnStart": "true",
"SHARE_PROJECT_CONFIGURATION_FILES": "true",
"WebServerToolWindowFactoryState": "false"
}
}]]></component>
<component name="RecentsManager">
<key name="CreateClassDialog.RecentsKey">
<recent name="org.leolo.map.osm.extract.model" />
</key>
<key name="MoveFile.RECENT_KEYS">
<recent name="D:\IdeaProjects\OSM extract" />
<recent name="D:\IdeaProjects\OSM extract\src\main\resources" />
</key>
<key name="MoveClassesOrPackagesDialog.RECENTS_KEY">
<recent name="org.leolo.map.osm.extract.model" />
</key>
</component>
<component name="RunManager" selected="Application.Main">
<configuration name="Main" type="Application" factoryName="Application">
<option name="MAIN_CLASS_NAME" value="org.leolo.map.osm.extract.ExtractElement" />
<module name="OSM extract" />
<option name="PROGRAM_PARAMETERS" value="-d &quot;D:\Downloads\isle-of-man-latest.osm.pbf&quot; -a action.xml --format plain" />
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<method v="2">
<option name="Make" enabled="true" />
</method>
</configuration>
<configuration name="OSM-extract [clean,compile]" type="MavenRunConfiguration" factoryName="Maven" nameIsGenerated="true">
<MavenSettings>
<option name="myGeneralSettings" />
<option name="myRunnerSettings" />
<option name="myRunnerParameters">
<MavenRunnerParameters>
<option name="profiles">
<set />
</option>
<option name="goals">
<list>
<option value="clean" />
<option value="compile" />
</list>
</option>
<option name="pomFileName" />
<option name="profilesMap">
<map />
</option>
<option name="resolveToWorkspace" value="false" />
<option name="workingDirPath" value="$PROJECT_DIR$" />
</MavenRunnerParameters>
</option>
</MavenSettings>
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<method v="2" />
</configuration>
<configuration name="OSM-extract [clean,compile,test]" type="MavenRunConfiguration" factoryName="Maven" nameIsGenerated="true">
<MavenSettings>
<option name="myGeneralSettings" />
<option name="myRunnerSettings" />
<option name="myRunnerParameters">
<MavenRunnerParameters>
<option name="profiles">
<set />
</option>
<option name="goals">
<list>
<option value="clean" />
<option value="compile" />
<option value="test" />
</list>
</option>
<option name="pomFileName" />
<option name="profilesMap">
<map />
</option>
<option name="resolveToWorkspace" value="false" />
<option name="workingDirPath" value="$PROJECT_DIR$" />
</MavenRunnerParameters>
</option>
</MavenSettings>
<extension name="software.aws.toolkits.jetbrains.core.execution.JavaAwsConnectionExtension">
<option name="credential" />
<option name="region" />
<option name="useCurrentConnection" value="false" />
</extension>
<method v="2" />
</configuration>
<list>
<item itemvalue="Application.Main" />
<item itemvalue="Maven.OSM-extract [clean,compile,test]" />
<item itemvalue="Maven.OSM-extract [clean,compile]" />
</list>
</component>
<component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
<component name="TaskManager">
<task active="true" id="Default" summary="Default task">
<changelist id="aada2b3d-ba01-45e9-89c7-a596447b137a" name="Changes" comment="" />
<created>1663575347472</created>
<option name="number" value="Default" />
<option name="presentableId" value="Default" />
<updated>1663575347472</updated>
<workItem from="1663575350840" duration="42097000" />
</task>
<servers />
</component>
<component name="TypeScriptGeneratedFilesManager">
<option name="version" value="3" />
</component>
<component name="Vcs.Log.Tabs.Properties">
<option name="TAB_STATES">
<map>
<entry key="MAIN">
<value>
<State />
</value>
</entry>
</map>
</option>
</component>
<component name="XDebuggerManager">
<watches-manager>
<configuration name="Application">
<watch expression="((Long)((Hashtable.Entry)((Hashtable)((ExtractElement)this).allEntry).entrySet().toArray()[1]).getKey()).value" custom="java.lang.Long,org.leolo.map.osm.extract.ExtractElement,java.util.Hashtable.Entry,java.util.Hashtable" />
</configuration>
</watches-manager>
</component>
</project>

2
Jenkinsfile vendored

@ -17,7 +17,7 @@ pipeline{
}
stage('Build'){
steps{
sh 'mvn -Dmaven.test.failure.ignore=true install test'
sh 'mvn -Dmaven.test.failure.ignore=true install'
}
post{
success{

21
examples/simple-action.xml

@ -0,0 +1,21 @@
<?xml version="1.0" encoding="utf-8" ?>
<actions>
<properties>
<lang>
<default>en</default>
<other>de</other>
<other>es</other>
<other>fr</other>
<other>ja</other>
</lang>
<tags>
<display>short_name</display>
<display>website</display>
</tags>
</properties>
<extracts>
<relation id="9936954"/>
<way id="79289788"/>
<node id="6720935522"/>
</extracts>
</actions>

43
pom.xml

@ -7,7 +7,28 @@
<groupId>org.leolo</groupId>
<artifactId>OSM-extract</artifactId>
<version>1.0-SNAPSHOT</version>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<resources>
<resource>
<directory>src/main/resources</directory>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>3.0.0-M4</version>
<configuration>
<includes>
<include>org.leolo.map.osm.extract.test.**</include>
</includes>
</configuration>
</plugin>
</plugins>
</build>
<properties>
<maven.compiler.source>18</maven.compiler.source>
<maven.compiler.target>18</maven.compiler.target>
@ -30,11 +51,6 @@
<version>3.0.5</version>
</dependency>
<dependency>
<groupId>org.openstreetmap.pbf</groupId>
<artifactId>osmpbf</artifactId>
<version>1.5.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.13.2</version>
@ -44,6 +60,23 @@
<artifactId>junit-jupiter-api</artifactId>
<version>5.9.0</version>
</dependency>
<dependency>
<groupId>de.topobyte</groupId>
<artifactId>osm4j-pbf</artifactId>
<version>1.2.0</version>
</dependency>
</dependencies>
<repositories>
<repository>
<id>topobyte</id>
<name>topobyte</name>
<url>https://mvn.topobyte.de</url>
</repository>
<repository>
<id>Slimjars</id>
<name>Slimjars</name>
<url>https://mvn.slimjars.com</url>
</repository>
</repositories>
</project>

286
src/main/java/org/leolo/map/osm/extract/ExtractElement.java

@ -1,18 +1,19 @@
package org.leolo.map.osm.extract;
import crosby.binary.BinaryParser;
import crosby.binary.Osmformat;
import crosby.binary.file.BlockInputStream;
import de.topobyte.osm4j.core.access.*;
import de.topobyte.osm4j.core.model.iface.*;
import de.topobyte.osm4j.pbf.seq.PbfReader;
import org.apache.commons.cli.*;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.leolo.map.osm.extract.model.*;
import org.leolo.map.osm.extract.util.InputFile;
import org.leolo.map.osm.extract.util.InputUtil;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
import java.io.*;
import java.util.*;
import java.util.concurrent.*;
public class ExtractElement {
@ -58,13 +59,33 @@ public class ExtractElement {
.required(true)
.desc("output format")
.build();
Option httpUserName = Option.builder()
.longOpt("http-user")
.argName("user name")
.hasArg(true)
.optionalArg(true)
.desc("User name sent during HTTP request")
.build();
Option httpPassword = Option.builder()
.longOpt("http-password")
.argName("passsword")
.hasArg(true)
.optionalArg(true)
.desc("Password sent during HTTP request")
.build();
}
private Map<Long, MapEntry> allEntry = new Hashtable<>();
private Map<Long, Relation> relations = new Hashtable<>();
private Map<Long, Way> ways = new Hashtable<>();
private Map<Long, Node> nodes = new Hashtable<>();
private Set<Integer> strings = new HashSet<>();
private HashSet<Long> pendingRelations = new HashSet<>();
private HashSet<Long> pendingWays = new HashSet<>();
private HashSet<Long> pendingNodes = new HashSet<>();
private CliOpt cliOpt = new CliOpt();
@ -79,6 +100,8 @@ public class ExtractElement {
String outputFilePath = null;
String actionFilePath;
String format;
InputFile dbFile;
InputFile actionFile;
try {
CommandLine cmd = parser.parse(getOptions(), args);
if(args.length == 0 || cmd.hasOption(cliOpt.help)){
@ -109,20 +132,25 @@ public class ExtractElement {
System.exit(2);
}
PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath));
File dbFile = new File(dbFilePath);
if(!dbFile.exists() || !dbFile.canRead()){
log.atError().log("Unable to read db file {}", dbFilePath);
System.exit(2);
}
File actionFile = new File(actionFilePath);
if(!actionFile.exists() || !actionFile.canRead()){
log.atError().log("Unable to read action file {}", actionFilePath);
System.exit(2);
}
dbFile = InputUtil.getInputFile(dbFilePath);
actionFile = InputUtil.getInputFile(actionFilePath);
log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000);
log.atInfo().log("Action file size {}kB", actionFile.length()/1_000);
doExtract(dbFile, actionFile, outputStream);
try {
doExtract(dbFile, actionFile);
}catch(IOException e){
log.atError().withThrowable(e).log("Exception when extracting the data!");
}
Class formatClass = FormatterManager.getInstance().get(format);
OutputFormat of;
try{
of = (OutputFormat) formatClass.getConstructor().newInstance();
}catch (Exception e){
log.atFatal().withThrowable(e).log("Unable to create an instance of OutputFormat");
System.exit(1);
return;
}
doOutput(actionFile, outputStream, of);
} catch(MissingOptionException|MissingArgumentException moe) {
System.out.println("FATAL ERROR : "+moe.getMessage());
printHelp(1);
@ -130,13 +158,13 @@ public class ExtractElement {
System.err.println("Fatal error - unable to parse command line input!");
e.printStackTrace();
System.exit(1);
} catch (FileNotFoundException e) {
} catch (IOException e) {
log.atError().withThrowable(e).log("Unable to locate the specified file");
System.exit(1);
}
}
private void doExtract(File dbFile, File actionFile, PrintStream outputStream) {
private void doExtract(InputFile dbFile, InputFile actionFile) throws IOException{
//Step 1: parse the actionFile
ActionFile af;
try {
@ -146,21 +174,226 @@ public class ExtractElement {
System.exit(3);
return;
}
expandRelations(dbFile, af, af.getRelation());
processWay(dbFile, af);
processNode(dbFile, af);
doSearch(dbFile, af);
for(ExtractItem ei:af.getRelation()){
pendingRelations.add(ei.getId());
}
for(ExtractItem ei:af.getWay()){
pendingWays.add(ei.getId());
}
for(ExtractItem ei:af.getNode()){
pendingNodes.add(ei.getId());
}
expandRelations(dbFile, 1);
processWay(dbFile);
processNode(dbFile);
log.atInfo().log("Done processing input!");
}
private void doOutput(InputFile actionFile, PrintStream outputStream, OutputFormat of) {
ExtractResult extractResult = new ExtractResult(allEntry, relations, ways, nodes);
PrintWriter out = new PrintWriter(outputStream);
of.writeHeader(out);
for(long id:relations.keySet()){
of.writeRelation(out, id, extractResult);
}
of.writeTailer(out);
out.flush();
out.close();
}
/**
* Perform search as defined in the action file, and add them to relevant pending items
* @param dbFile The database file going to be searched
* @param af The file which defines the action to be performed
*/
private void doSearch(InputFile dbFile, ActionFile af) throws IOException{
if(af.getSearches().isEmpty()){
return;
}
OsmReader reader = new PbfReader(dbFile.getInputStream(), true);
reader.setHandler(new OsmHandler() {
@Override
public void handle(OsmBounds osmBounds) throws IOException {
}
@Override
public void handle(OsmNode osmNode) throws IOException {
for(SearchItem si:af.getSearches()){
if(si.matchNode(osmNode)){
log.atInfo().log("Matched Node {}", osmNode.getId());
ExtractItem ei = new ExtractItem();
ei.setId(osmNode.getId());
af.addNode(ei);
}
}
}
@Override
public void handle(OsmWay osmWay) throws IOException {
for(SearchItem si:af.getSearches()){
if(si.matchWay(osmWay)){
log.atInfo().log("Matched Way {}", osmWay.getId());
ExtractItem ei = new ExtractItem();
ei.setId(osmWay.getId());
af.addWay(ei);
}
}
}
@Override
public void handle(OsmRelation osmRelation) throws IOException {
for(SearchItem si:af.getSearches()){
if(si.matchRelation(osmRelation)){
log.atInfo().log("Matched Relation {}", osmRelation.getId());
ExtractItem ei = new ExtractItem();
ei.setId(osmRelation.getId());
af.addRelation(ei);
}
}
}
@Override
public void complete() throws IOException {
log.atInfo().log("Finish reading the DB file");
}
});
try {
reader.read();
} catch (OsmInputException e) {
log.atError().withThrowable(e).log("Error when reading the input!");
}
}
private void processNode(File dbFile, ActionFile af) {
private void processNode(InputFile dbFile) throws IOException{
if(pendingNodes.isEmpty())
return;
log.atInfo().log("Processing {} node(s)", pendingNodes.size());
OsmReader reader = new PbfReader(dbFile.getInputStream(), true);
reader.setHandler(new DefaultOsmHandler() {
@Override
public void handle(OsmNode node) throws IOException {
super.handle(node);
if(pendingNodes.contains(node.getId())){
log.atDebug().log("Found node #{}", node.getId());
Node n = new Node(node.getLatitude(), node.getLongitude());
for(int i=0;i<node.getNumberOfTags();i++){
OsmTag t = node.getTag(i);
log.atDebug().log("{}->{}", t.getKey(), t.getValue());
n.putTag(t.getKey(), t.getValue());
}
allEntry.put(node.getId(), n);
nodes.put(node.getId(), n);
}
}
@Override
public void complete() throws IOException {
super.complete();
log.atInfo().log("Finish reading the DB file");
}
});
try {
reader.read();
} catch (OsmInputException e) {
log.atError().withThrowable(e).log("Error when reading the input!");
}
}
private void processWay(File dbFile, ActionFile af) {
private void processWay(InputFile dbFile) throws IOException{
if(pendingWays.isEmpty())
return;
log.atInfo().log("Processing {} way(s)", pendingWays.size());
OsmReader reader = new PbfReader(dbFile.getInputStream(), true);
reader.setHandler(new DefaultOsmHandler() {
@Override
public void handle(OsmWay w) throws IOException {
super.handle(w);
if(pendingWays.contains(w.getId())){
Way way = new Way();
log.atDebug().log("Found way #{}", w.getId());
for(int i=0;i<w.getNumberOfNodes();i++){
way.addMember(w.getNodeId(i));
pendingNodes.add(w.getNodeId(i));
}
for(int i=0;i<w.getNumberOfTags();i++){
OsmTag tag = w.getTag(i);
log.atDebug().log("{}->{}", tag.getKey(), tag.getValue());
way.putTag(tag.getKey(), tag.getValue());
}
allEntry.put(w.getId(), way);
ways.put(w.getId(), way);
}
}
@Override
public void complete() throws IOException {
super.complete();
log.atInfo().log("Finish reading the DB file");
}
});
try {
reader.read();
} catch (OsmInputException e) {
log.atError().withThrowable(e).log("Error when reading the input!");
}
}
private void expandRelations(File dbFile, ActionFile af, Collection<ExtractItem> relations) {
private void expandRelations(InputFile dbFile,final int round) throws IOException{
if(pendingRelations.isEmpty()) {
log.atInfo().log("No relation to be expanded");
return;
}
HashSet<Long> nextRound = new HashSet<>();
log.atInfo().log("Expanding relations round {} with {} relation(s)", round, pendingRelations.size());
OsmReader reader = new PbfReader(dbFile.getInputStream(), true);
reader.setHandler(new DefaultOsmHandler() {
@Override
public void handle(OsmRelation relation) throws IOException {
super.handle(relation);
if(pendingRelations.contains(relation.getId())){
log.atDebug().log("Relation {} found with {} member(s) and {} tag(s).",
relation.getId(),
relation.getNumberOfMembers(),
relation.getNumberOfTags()
);
Relation rel = new Relation();
for(int i=0;i<relation.getNumberOfMembers();i++){
OsmRelationMember orm = relation.getMember(i);
switch (orm.getType()){
case Relation -> nextRound.add(orm.getId());
case Way -> pendingWays.add(orm.getId());
case Node -> pendingNodes.add(orm.getId());
}
rel.addMember(orm.getId());
}
for(int i=0;i<relation.getNumberOfTags();i++){
OsmTag tag = relation.getTag(i);
log.atDebug().log("{}->{}", tag.getKey(), tag.getValue());
rel.putTag(tag.getKey(), tag.getValue());
}
allEntry.put(relation.getId(), rel);
relations.put(relation.getId(), rel);
}
}
@Override
public void complete() throws IOException {
super.complete();
log.atInfo().log("Finish reading round {} when expanding relation", round);
}
});
try {
reader.read();
} catch (OsmInputException e) {
log.atError().withThrowable(e).log("Error when reading the input!");
}
log.atInfo().log("There are {} relations to be expanded in next round", nextRound.size());
pendingRelations = nextRound;
if(pendingRelations.size()>0){
expandRelations(dbFile, round+1);
}
}
private void printHelp(int returnValue){
@ -179,6 +412,9 @@ public class ExtractElement {
options.addOption(cliOpt.actionFile);
options.addOption(cliOpt.help);
options.addOption(cliOpt.outputFormat);
//Commented first, need to find some way to pass the parameters to the function
// options.addOption(cliOpt.httpUserName);
// options.addOption(cliOpt.httpPassword);
}
return options;
}

40
src/main/java/org/leolo/map/osm/extract/ExtractResult.java

@ -0,0 +1,40 @@
package org.leolo.map.osm.extract;
import org.leolo.map.osm.extract.model.MapEntry;
import org.leolo.map.osm.extract.model.Node;
import org.leolo.map.osm.extract.model.Relation;
import org.leolo.map.osm.extract.model.Way;
import java.util.Hashtable;
import java.util.Map;
public class ExtractResult {
private Map<Long, MapEntry> allEntry = new Hashtable<>();
private Map<Long, Relation> relations = new Hashtable<>();
private Map<Long, Way> ways = new Hashtable<>();
private Map<Long, Node> nodes = new Hashtable<>();
public ExtractResult(Map<Long, MapEntry> allEntry, Map<Long, Relation> relations, Map<Long, Way> ways, Map<Long, Node> nodes) {
this.allEntry = allEntry;
this.relations = relations;
this.ways = ways;
this.nodes = nodes;
}
public Map<Long, MapEntry> getAllEntry() {
return allEntry;
}
public Map<Long, Relation> getRelations() {
return relations;
}
public Map<Long, Way> getWays() {
return ways;
}
public Map<Long, Node> getNodes() {
return nodes;
}
}

8
src/main/java/org/leolo/map/osm/extract/OutputFormat.java

@ -1,4 +1,12 @@
package org.leolo.map.osm.extract;
import java.io.PrintWriter;
public interface OutputFormat {
public void writeHeader(PrintWriter output);
public void writeTailer(PrintWriter output);
public void writeRelation(PrintWriter output,long id, ExtractResult extractResult);
public void writeWay(PrintWriter output,long id, ExtractResult extractResult);
public void writeNode(PrintWriter output,long id, ExtractResult extractResult);
}

12
src/main/java/org/leolo/map/osm/extract/SearchProvider.java

@ -0,0 +1,12 @@
package org.leolo.map.osm.extract;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
public @interface SearchProvider {
String[] searchKey();
}

112
src/main/java/org/leolo/map/osm/extract/SearchProviderManager.java

@ -0,0 +1,112 @@
package org.leolo.map.osm.extract;
import eu.infomas.annotation.AnnotationDetector;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.leolo.map.osm.extract.model.SearchItem;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.util.*;
import java.util.function.BiConsumer;
public class SearchProviderManager {
private static SearchProviderManager instance;
private Logger log = LogManager.getLogger();
private Hashtable<String, Class> items = new Hashtable<>();
public static synchronized SearchProviderManager getInstance(){
if(instance==null){
instance = new SearchProviderManager();
}
return instance;
}
private SearchProviderManager(){
AnnotationDetector.TypeReporter reporter = new AnnotationDetector.TypeReporter() {
@Override
public void reportTypeAnnotation(Class<? extends Annotation> annoClass, String clazz) {
log.atDebug().log("Class {} has annotation {}", clazz, annoClass.getName());
try {
Class typeClass = Class.forName(clazz);
if(SearchItem.class.isAssignableFrom(typeClass)){
log.atDebug().log("Class {} is an output formatter", clazz);
SearchProvider of = (SearchProvider) typeClass.getAnnotation(SearchProvider.class);
for(String format: of.searchKey()){
if(items.containsKey(format)){
log.atWarn().log("Duplicated search type {}. Shared by (at least) {} and {}",
format,
typeClass.getName(),
items.get(format).getName()
);
}else{
items.put(format, typeClass);
log.atInfo().log("Registering class {} for search type {}", typeClass, format);
}
}
}else{
log.atError().log("Class {} does not implements OutputFormat but have annotation @OutputFormatter", clazz);
}
} catch (ClassNotFoundException e) {
log.atError().withThrowable(e).log(e.getMessage());
}
}
@Override
public Class<? extends Annotation>[] annotations() {
return new Class[]{SearchProvider.class};
}
};
final AnnotationDetector cf = new AnnotationDetector(reporter);
try {
cf.detect();
} catch (IOException e) {
log.atError().withThrowable(e).log(e.getMessage());
}
}
public int size() {
return items.size();
}
public boolean isEmpty() {
return items.isEmpty();
}
public Enumeration<String> keys() {
return items.keys();
}
public boolean containsKey(Object key) {
return items.containsKey(key);
}
public Class get(Object key) {
return items.get(key);
}
public Set<String> keySet() {
return items.keySet();
}
public void forEach(BiConsumer<? super String, ? super Class> action) {
items.forEach(action);
}
public int getOutputFormatterCount(){
HashSet<Class> set = new HashSet<>();
for(Class clazz:items.values()){
set.add(clazz);
}
return set.size();
}
public int getSearchProviderCount(){
HashSet<Class> set = new HashSet<>();
for(Class clazz:items.values()){
set.add(clazz);
}
return set.size();
}
}

82
src/main/java/org/leolo/map/osm/extract/format/KMLFormatter.java

@ -1,8 +1,90 @@
package org.leolo.map.osm.extract.format;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.leolo.map.osm.extract.ExtractResult;
import org.leolo.map.osm.extract.OutputFormat;
import org.leolo.map.osm.extract.OutputFormatter;
import org.leolo.map.osm.extract.model.MapEntry;
import org.leolo.map.osm.extract.model.Node;
import org.leolo.map.osm.extract.model.Relation;
import org.leolo.map.osm.extract.model.Way;
import java.io.PrintWriter;
@OutputFormatter(formatName = {"kml","application/vnd.google-earth.kml+xml"})
public class KMLFormatter implements OutputFormat {
private Logger log = LogManager.getLogger();
@Override
public void writeHeader(PrintWriter output) {
output.print("<?xml version='1.0'?><kml xmlns='http://www.opengis.net/kml/2.2' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='https://schemas.opengis.net/kml/2.3/ogckml23.xsd'>");
output.print("<Document>");
output.print("<Style id='sty_0'>");
output.print("<LineStyle><color>501400FF</color><width>3</width></LineStyle>");
output.print("</Style>");
output.print("<Style id='sty_1'><ListStyle><listItemType>checkHideChildren</listItemType></ListStyle></Style>");
}
@Override
public void writeTailer(PrintWriter output) {
output.print("</Document></kml>");
}
@Override
public void writeRelation(PrintWriter output, long id, ExtractResult extractResult) {
Relation rel = extractResult.getRelations().get(id);
output.print(getRelationString(id, extractResult));
}
private CharSequence getRelationString(long id, ExtractResult extractResult){
Relation rel = extractResult.getRelations().get(id);
StringBuffer sb = new StringBuffer();
sb.append("<Folder id='rel_").append(Long.toHexString(id)).append("'>");
sb.append("<styleUrl>#sty_1</styleUrl>");
sb.append("<name><![CDATA[").append(rel.getTag("name")).append("<]]></name>");
sb.append("<description><![CDATA[<table><tr><th>Tag Name</th><th>Tag Value</th></tr>");
rel.tagSet().stream().forEach((tagName)->{
StringBuilder tag = new StringBuilder();
tag.append("<tr><td>").append(tagName).append("</td><td>").append(rel.getTag(tagName)).append("</td>");
sb.append(tag);
});
sb.append("</table>]]></description>");
rel.parallelStream().forEach((member)->{
MapEntry me = extractResult.getAllEntry().get(member);
if(me==null){
log.atWarn().log("Missing entry {}", id);
return;
}
if(me instanceof Relation){
//Calling same function to create nested folder
sb.append(getRelationString(member, extractResult));
}else if(me instanceof Way){
//Handling nameless way
StringBuffer wayOut = new StringBuffer();
Way way = (Way) me;
wayOut.append("<Placemark><styleUrl>#sty_0</styleUrl><LineString id='rel_").append(Long.toHexString(id))
.append("_w_").append(Long.toHexString(member)).append("'>");
wayOut.append("<coordinates>");
for(int i=0;i<way.memberCount();i++){
Node node = extractResult.getNodes().get(way.getMember(i));
wayOut.append(node.getLon()).append(",").append(node.getLat()).append(" ");
}
wayOut.append("</coordinates>");
wayOut.append("</LineString></Placemark>");
sb.append(wayOut);
}
});
sb.append("</Folder>");
return sb;
}
@Override
public void writeWay(PrintWriter output, long id, ExtractResult extractResult) {
}
@Override
public void writeNode(PrintWriter output, long id, ExtractResult extractResult) {
}
}

4
src/main/java/org/leolo/map/osm/extract/format/PlainFormatter.java

@ -3,6 +3,6 @@ package org.leolo.map.osm.extract.format;
import org.leolo.map.osm.extract.OutputFormat;
import org.leolo.map.osm.extract.OutputFormatter;
@OutputFormatter(formatName = "plain")
public class PlainFormatter implements OutputFormat {
//@OutputFormatter(formatName = "plain")
public class PlainFormatter{// implements OutputFormat {
}

24
src/main/java/org/leolo/map/osm/extract/format/SimpleStringSearchProvider.java

@ -0,0 +1,24 @@
package org.leolo.map.osm.extract.format;
import org.leolo.map.osm.extract.SearchProvider;
import org.leolo.map.osm.extract.model.SearchItem;
@SearchProvider(searchKey = "name")
public class SimpleStringSearchProvider extends SearchItem {
private String searchKey;
@Override
public boolean matchString(String target) {
return searchKey.equalsIgnoreCase(target.strip());
}
@Override
public void setSearchKey(String searchKey) {
this.searchKey = searchKey.strip();
}
@Override public String toString(){
return "SimpleString["+searchKey+"]";
}
}

63
src/main/java/org/leolo/map/osm/extract/model/ActionFile.java

@ -2,17 +2,21 @@ package org.leolo.map.osm.extract.model;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.leolo.map.osm.extract.SearchProviderManager;
import org.leolo.map.osm.extract.util.InputFile;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
import org.xml.sax.SAXException;
import javax.management.ReflectionException;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.InvocationTargetException;
import java.util.*;
public class ActionFile {
@ -23,10 +27,15 @@ public class ActionFile {
private Map<Long,ExtractItem> way = new Hashtable<>();
private Map<Long,ExtractItem> node = new Hashtable<>();
public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException {
log.atInfo().log("Parsing action file {}", path.getAbsolutePath());
private String baseLanguage;
private HashSet<String> otherLanguage = new HashSet<>();
private Vector<SearchItem> searches = new Vector<>();
public ActionFile(InputFile path) throws IOException, SAXException, ParserConfigurationException {
log.atInfo().log("Parsing action file");
DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder();
Document doc = builder.parse(path);
Document doc = builder.parse(path.getInputStream());
doc.getDocumentElement().normalize();
Element rootElement = doc.getDocumentElement();
log.atDebug().log("Root element : {}", rootElement.getTagName());
@ -85,6 +94,42 @@ public class ActionFile {
continue;
}
}
NodeList languages = rootElement.getElementsByTagName("language");
if(languages.getLength()>0){
NodeList children = languages.item(0).getChildNodes();
for(int i=0;i<children.getLength();i++) {
Node child = children.item(i);
if("base".equals(child.getNodeName())){
this.baseLanguage = child.getTextContent();
}else if("other".equals(child.getNodeName())){
this.otherLanguage.add(child.getTextContent());
}
}
}
NodeList searchesNodeList = rootElement.getElementsByTagName("searches");
if(searchesNodeList.getLength()>0){
NodeList searches = searchesNodeList.item(0).getChildNodes();
for(int i=0;i<searches.getLength();i++){
Node child = searches.item(i);
if(child==null||"#text".equals(child.getNodeName()) || "#comment".equals(child.getNodeName())){
continue;
}
log.atDebug().log("Search type {}; search key: {}", child.getNodeName(), child.getTextContent());
Class clazz = SearchProviderManager.getInstance().get(child.getNodeName());
if(clazz==null){
log.atError().log("Unknown search type {}", child.getNodeName());
}else{
try {
SearchItem si = (SearchItem) clazz.getConstructor().newInstance();
si.setSearchKey(child.getTextContent());
si.setActionFile(this);
this.searches.add(si);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
}
}
@ -118,4 +163,16 @@ public class ActionFile {
if(!way.containsKey(ei.getId()))
way.put(ei.getId(), ei);
}
public String getBaseLanguage() {
return baseLanguage;
}
public HashSet<String> getOtherLanguage() {
return otherLanguage;
}
public Vector<SearchItem> getSearches() {
return searches;
}
}

3
src/main/java/org/leolo/map/osm/extract/model/ExtractItem.java

@ -2,11 +2,12 @@ package org.leolo.map.osm.extract.model;
import java.util.Hashtable;
import java.util.Map;
import java.util.TreeMap;
public class ExtractItem implements Comparable<ExtractItem>{
private long id;
private String label;
private Map<String, Object> parameter = new Hashtable<>();
private Map<String, Object> parameter = new TreeMap<>();
private boolean fromFile = false;
public boolean isFromFile() {

6
src/main/java/org/leolo/map/osm/extract/model/InputType.java

@ -0,0 +1,6 @@
package org.leolo.map.osm.extract.model;
public enum InputType {
FILE,
URL;
}

4
src/main/java/org/leolo/map/osm/extract/model/MapEntry.java

@ -3,9 +3,10 @@ package org.leolo.map.osm.extract.model;
import java.util.Hashtable;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
public class MapEntry {
private Map<String, String> tags = new Hashtable<>();
private Map<String, String> tags = new TreeMap<>();
public int tagCount() {
return tags.size();
@ -30,4 +31,5 @@ public class MapEntry {
public Set<String> tagSet() {
return tags.keySet();
}
}

16
src/main/java/org/leolo/map/osm/extract/model/Node.java

@ -9,4 +9,20 @@ public class Node extends MapEntry{
this.lat = lat;
this.lon = lon;
}
public double getLat() {
return lat;
}
public void setLat(double lat) {
this.lat = lat;
}
public double getLon() {
return lon;
}
public void setLon(double lon) {
this.lon = lon;
}
}

9
src/main/java/org/leolo/map/osm/extract/model/Relation.java

@ -1,6 +1,7 @@
package org.leolo.map.osm.extract.model;
import java.util.*;
import java.util.stream.Stream;
public class Relation extends MapEntry{
private List<Long> members = new Vector<>();
@ -25,4 +26,12 @@ public class Relation extends MapEntry{
public Long getMember(int index) {
return members.get(index);
}
public Stream<Long> stream() {
return members.stream();
}
public Stream<Long> parallelStream() {
return members.parallelStream();
}
}

68
src/main/java/org/leolo/map/osm/extract/model/SearchItem.java

@ -0,0 +1,68 @@
package org.leolo.map.osm.extract.model;
import de.topobyte.osm4j.core.model.iface.OsmEntity;
import de.topobyte.osm4j.core.model.iface.OsmNode;
import de.topobyte.osm4j.core.model.iface.OsmRelation;
import de.topobyte.osm4j.core.model.iface.OsmWay;
import java.util.HashSet;
import java.util.Set;
public abstract class SearchItem {
private ActionFile af;
private HashSet<String> tagName = new HashSet<>();
public SearchItem(ActionFile af){
this.af = af;
setTags();
}
public SearchItem(){
//Empty constructor
}
protected final void setActionFile(ActionFile af){
this.af = af;
setTags();
}
private void setTags(){
tagName.add("name:"+af.getBaseLanguage());
for(String lang:af.getOtherLanguage()){
tagName.add("name:"+lang);
}
}
public abstract boolean matchString(String target);
public abstract void setSearchKey(String searchKey);
private boolean matchEntity(OsmEntity entity){
for(int i=0;i<entity.getNumberOfTags();i++){
if("name".equalsIgnoreCase(entity.getTag(i).getKey()) && matchString(entity.getTag(i).getValue())){
return true;
}
if(af != null){
for(String tag:tagName){
if(tag.equalsIgnoreCase(entity.getTag(i).getKey()) && matchString(entity.getTag(i).getValue())){
return true;
}
}
}
}
return false;
}
public boolean matchNode(OsmNode node){
return matchEntity(node);
}
public boolean matchWay(OsmWay way){
return matchEntity(way);
}
public boolean matchRelation(OsmRelation relation){
return matchEntity(relation);
}
}

92
src/main/java/org/leolo/map/osm/extract/util/InputFile.java

@ -0,0 +1,92 @@
package org.leolo.map.osm.extract.util;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.*;
import java.net.MalformedURLException;
import java.net.URL;
public abstract class InputFile {
public abstract long getSize();
public abstract InputStream getInputStream() throws IOException;
public long length() {
return getSize();
}
public boolean isValid(){
return false;
}
}
class FileInputFile extends InputFile{
private File f;
FileInputFile(File f){
this.f = f;
}
FileInputFile(String f){
this.f = new File(f);
}
@Override
public long getSize(){
return f.length();
}
@Override
public InputStream getInputStream() throws FileNotFoundException {
return new FileInputStream(f);
}
@Override
public boolean isValid(){
return f.exists() && f.canRead();
}
}
class URLInputFile extends InputFile{
private Logger log = LogManager.getLogger();
private boolean ready = false;
private byte[] data;
public static final int READ_BLOCK_SIZE = 4096;
URLInputFile(String url) throws MalformedURLException, IOException {
this(new URL(url));
}
URLInputFile(URL url) throws IOException{
log.atInfo().log("Connecting to {}", url);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
InputStream is = url.openStream();
byte [] buffer = new byte[READ_BLOCK_SIZE];
int readSize;
while((readSize = is.read(buffer))>0){
baos.write(buffer, 0, readSize);
}
data = baos.toByteArray();
is.close();
baos.close();
ready = true;
}
@Override
public long getSize(){
return data.length;
}
@Override
public InputStream getInputStream() throws IOException {
return new ByteArrayInputStream(data);
}
@Override
public boolean isValid() {
return ready;
}
}

27
src/main/java/org/leolo/map/osm/extract/util/InputUtil.java

@ -0,0 +1,27 @@
package org.leolo.map.osm.extract.util;
import org.leolo.map.osm.extract.model.InputType;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Locale;
public class InputUtil {
public static InputType identifyInputType(String str){
try {
URL url = new URL(str);
} catch (Exception e) {
return InputType.FILE;
}
return InputType.URL;
}
public static InputFile getInputFile(String path) throws IOException {
if(identifyInputType(path)==InputType.FILE){
return new FileInputFile(path);
}else{
return new URLInputFile(path);
}
}
}

14
src/test/java/MainTest.java

@ -1,14 +0,0 @@
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import static org.junit.jupiter.api.Assertions.*;
public class MainTest {
@Test void test1(){
}
}

50
src/test/java/org/leolo/map/osm/extract/test/InputTypeTest.java

@ -0,0 +1,50 @@
package org.leolo.map.osm.extract.test;
import org.junit.jupiter.api.Test;
import org.leolo.map.osm.extract.model.InputType;
import org.leolo.map.osm.extract.util.InputUtil;
import static org.junit.jupiter.api.Assertions.*;
public class InputTypeTest {
@Test void testFilePath(){
assertEquals(InputUtil.identifyInputType("abc.xml"), InputType.FILE);
assertEquals(InputUtil.identifyInputType("../abc.xml"), InputType.FILE);
assertEquals(InputUtil.identifyInputType("./abc.xml"), InputType.FILE);
assertEquals(InputUtil.identifyInputType("..\\abc.xml"), InputType.FILE);
assertEquals(InputUtil.identifyInputType("/tmp/abc.xml"), InputType.FILE);
}
@Test void testUPCPath(){
assertEquals(InputUtil.identifyInputType("\\\\somecomputer\\somefolder\\abc.xml"), InputType.FILE);
}
@Test void testURL(){
assertEquals(InputUtil.identifyInputType("http://www.example.com"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://www.example.com"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a@www.example.com"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a@www.example.com"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://www.example.com:8521"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://www.example.com:8521"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://www.example.com/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://www.example.com/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a@www.example.com/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a@www.example.com/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://www.example.com:8521/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://www.example.com:8521/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a@www.example.com:8521/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a@www.example.com:8521/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("http://a:1234@www.example.com:8521/abc.xml"), InputType.URL);
assertEquals(InputUtil.identifyInputType("https://a:1234@www.example.com:8521/abc.xml"), InputType.URL);
}
}

23
src/test/java/org/leolo/map/osm/extract/test/MainTest.java

@ -0,0 +1,23 @@
package org.leolo.map.osm.extract.test;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.leolo.map.osm.extract.util.InputFile;
import org.leolo.map.osm.extract.util.InputUtil;
import java.io.IOException;
import java.util.ArrayList;
import static org.junit.jupiter.api.Assertions.*;
public class MainTest {
@Test void test1(){
System.out.println("Test executed.");
}
@Test void downloadFile() throws IOException {
InputFile inputFile = InputUtil.getInputFile("https://www.leolo.pro/sample");
assertEquals(inputFile.length(), 8192);
}
}
Loading…
Cancel
Save