From 6f5da10084f339820cd4cd90bb93100ca2a04460 Mon Sep 17 00:00:00 2001 From: gadelrab Date: Sat, 11 May 2019 20:41:03 +0200 Subject: [PATCH] allowing loading multiple KGs and step towards loading multiple multiple text spotters Creating ExplanationExtractorDemo --- .../src/main/java/config/Configuration.java | 226 +++++++++--------- .../java/extendedsldnf/EvaluatorFactory.java | 17 +- .../java/extendedsldnf/RecSLDEvaluator.java | 37 +-- .../datastructure/InputQuery.java | 42 ++-- .../datastructure/QueryExplanations.java | 39 ++- .../datastructure/TextualSource.java | 17 ++ .../main/java/text/FactSpottingConnector.java | 12 +- .../src/main/java/utils/DataUtils.java | 9 + 8 files changed, 209 insertions(+), 190 deletions(-) diff --git a/QueryRewriting/src/main/java/config/Configuration.java b/QueryRewriting/src/main/java/config/Configuration.java index 8ee8dbc..b87cdc7 100755 --- a/QueryRewriting/src/main/java/config/Configuration.java +++ b/QueryRewriting/src/main/java/config/Configuration.java @@ -29,10 +29,14 @@ public class Configuration extends org.deri.iris.Configuration { private static final String FACTS_FORMAT = "factsFormat"; private static final String PARTIAL_BINDING_TYPE = "partialBindingType"; private static final String SUSPECTS_FROM_KG = "suspectsFromKG"; - public static final String FACT_SPOTTING_CONF = "factSpotting.config"; + private static final String FACT_SPOTTING_CONF = "factSpotting.config"; private static final String EVALUATION_METHOD = "evalMethod"; private static final String MAX_EXPLANATIONS="maxExplanations"; private static final String MAX_RULE_NESTING="maxRuleNesting"; + private static final String KGS_DIR="kgsDir"; + + + private static Configuration instance; @@ -43,6 +47,10 @@ public class Configuration extends org.deri.iris.Configuration { */ private static boolean externalConfFile=false; private static ClassLoader classLoader; + /** + * FactSpotting Method + */ + FactSpotterFactory.SpottingMethod spottingMethod= FactSpotterFactory.SpottingMethod.NONE; /** * Extra configurations */ @@ -52,83 +60,11 @@ public class Configuration extends org.deri.iris.Configuration { private int maxExplanations=10; private int maxRuleNesting =3; private ClassLoader spottingConfFileClassLoader; - ; - - public void setExtraProp(Properties extraProp) { - this.extraProperties = extraProp; - } - - public boolean isSuspectsFromKG() { - return suspectsFromKG; - } - - public void setSuspectsFromKG(boolean suspectsFromKG) { - this.suspectsFromKG = suspectsFromKG; - } - - public void setSpottingConfFile(String spottingConfFile,ClassLoader classLoader) { - setSpottingConfFile(spottingConfFile); - setSpottingConfFileClassLoader(classLoader); - } - - public void setSpottingConfFile(String spottingConfFile) { - this.spottingConfFile = spottingConfFile; - - } - - - public void setSpottingConfFileClassLoader(ClassLoader spottingConfFileClassLoader) { - this.spottingConfFileClassLoader = spottingConfFileClassLoader; - } - - public String getSpottingConfFile() { - return spottingConfFile; - } - - public ClassLoader getSpottingConfFileClassLoader(){ - return spottingConfFileClassLoader; - } - - public void setEvaluationMethod(Enums.EvalMethod evaluationMethod) { - this.evaluationMethod = evaluationMethod; - } - - public Enums.EvalMethod getEvaluationMethod() { - return evaluationMethod; - } - - public int getMaxExplanations() { - return maxExplanations; - } - - public int getMaxRuleNesting() { - return maxRuleNesting; - } - - public void setMaxExplanations(int maxExplanations) { - this.maxExplanations = maxExplanations; - } - - public void setMaxRuleNesting(int maxRuleNesting) { - this.maxRuleNesting = maxRuleNesting; - } - - - public enum PartialBindingType {GREEDY,TEXT,NONE} - - public enum FactsFormat {RDF, IRIS;} - - //public enum TextCheckingMode{GROUNDED, PARTIAL, NONE, KG_BIND;} - /** - * FactSpotting Method - */ - FactSpotterFactory.SpottingMethod spottingMethod= FactSpotterFactory.SpottingMethod.NONE; - + private String kgsDirectory; /** * Files containing rules */ private List rulesFiles; - /** * Files containing facts */ @@ -137,24 +73,20 @@ public enum FactsFormat {RDF, IRIS;} * Files containing queries */ private List queriesFiles; - /** * The format of the input fact, either RDF triples or IRIS facts */ private FactsFormat factsFormat; - /** * The allowed facts to check, either fully grounded or partially */ private PartialBindingType partialBindingType; - /** * suspected in the KG so remove them from KG */ private boolean suspectsFromKG=false; - public Configuration(){ super(); super.evaluationStrategyFactory=new ExtendedSLDNFEvaluationStrategyFactory(); @@ -175,7 +107,7 @@ public static Configuration fromFile(String filename, boolean inResources,ClassL return fromStream(input); // try { - // configuration file loaded in resources + // configuration file loaded in resources // if (inResources) { // input = Configuration.class.getClassLoader().getResourceAsStream(filename); // @@ -214,19 +146,19 @@ public static Configuration fromFile(String filename, boolean inResources,ClassL public static InputStream loadFile(String filename, boolean inResources, ClassLoader classLoader) { InputStream input = null; if (inResources) - input = Configuration.class.getClassLoader().getResourceAsStream(filename); - else - if(classLoader!=null) - input=classLoader.getResourceAsStream(filename); - else - // configuration file form user - { - try { - input = new FileInputStream(filename); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } - } + input = Configuration.class.getClassLoader().getResourceAsStream(filename); + else + if(classLoader!=null) + input=classLoader.getResourceAsStream(filename); + else + // configuration file form user + { + try { + input = new FileInputStream(filename); + } catch (FileNotFoundException e) { + e.printStackTrace(); + } + } return input; @@ -247,6 +179,7 @@ public static Configuration fromStream(InputStream input) { //get the property value conf.setSpottingMethod(FactSpotterFactory.SpottingMethod.valueOf(prop.getProperty(SPOTTING_METHOD, "NONE"))); + conf.setKgsDirectory(prop.getProperty(SPOTTING_METHOD, "kgs")); conf.setRulesFiles(asList(prop.getProperty(RULES_FILES, ""))); conf.setFactsFiles(asList(prop.getProperty(FACTS_FILES, ""))); conf.setQueriesFiles(asList(prop.getProperty(QUERIES_FILES, ""))); @@ -271,7 +204,6 @@ public static Configuration fromStream(InputStream input) { return conf; } - public static Configuration fromFile(String filename) { return fromFile(filename,!externalConfFile,classLoader); } @@ -283,9 +215,6 @@ public synchronized static Configuration getInstance() { return instance; } - - - public synchronized static boolean setConfigurationFile(String file,ClassLoader remoteClassLoader){ if(file==null||confFileAlreadySet) return false; @@ -300,19 +229,86 @@ public synchronized static boolean setConfigurationFile(String file,ClassLoader } } + public static String getConfigurationFile() { + return configurationFile; + } - public void setSpottingMethod(FactSpotterFactory.SpottingMethod spottingMethod) { - this.spottingMethod = spottingMethod; + public String getKgsDirectory() { + return kgsDirectory; } - public void setRulesFiles(List rulesFiles) { - this.rulesFiles = rulesFiles; + public void setKgsDirectory(String kgsDirectory) { + this.kgsDirectory = kgsDirectory; + } + + public void setExtraProp(Properties extraProp) { + this.extraProperties = extraProp; + } + + //public enum TextCheckingMode{GROUNDED, PARTIAL, NONE, KG_BIND;} + + public boolean isSuspectsFromKG() { + return suspectsFromKG; + } + + public void setSuspectsFromKG(boolean suspectsFromKG) { + this.suspectsFromKG = suspectsFromKG; + } + + public void setSpottingConfFile(String spottingConfFile,ClassLoader classLoader) { + setSpottingConfFile(spottingConfFile); + setSpottingConfFileClassLoader(classLoader); + } + + public String getSpottingConfFile() { + return spottingConfFile; + } + + public void setSpottingConfFile(String spottingConfFile) { + this.spottingConfFile = spottingConfFile; + + } + + public ClassLoader getSpottingConfFileClassLoader(){ + return spottingConfFileClassLoader; + } + + public void setSpottingConfFileClassLoader(ClassLoader spottingConfFileClassLoader) { + this.spottingConfFileClassLoader = spottingConfFileClassLoader; + } + + public Enums.EvalMethod getEvaluationMethod() { + return evaluationMethod; + } + + public void setEvaluationMethod(Enums.EvalMethod evaluationMethod) { + this.evaluationMethod = evaluationMethod; + } + + public int getMaxExplanations() { + return maxExplanations; + } + + public void setMaxExplanations(int maxExplanations) { + this.maxExplanations = maxExplanations; + } + + public int getMaxRuleNesting() { + return maxRuleNesting; + } + + public void setMaxRuleNesting(int maxRuleNesting) { + this.maxRuleNesting = maxRuleNesting; } public List getRulesFiles() { return rulesFiles; } + public void setRulesFiles(List rulesFiles) { + this.rulesFiles = rulesFiles; + } + public List getFactsFiles() { return factsFiles; } @@ -321,34 +317,38 @@ public void setFactsFiles(List factsFiles) { this.factsFiles = factsFiles; } - public void setQueriesFiles(List queriesFiles) { - this.queriesFiles = queriesFiles; + public FactsFormat getFactsFormat() { + return factsFormat; } public void setFactsFormat(FactsFormat factsFormat) { this.factsFormat = factsFormat; } - public FactsFormat getFactsFormat() { - return factsFormat; - } - public FactSpotterFactory.SpottingMethod getSpottingMethod() { return spottingMethod; } + public void setSpottingMethod(FactSpotterFactory.SpottingMethod spottingMethod) { + this.spottingMethod = spottingMethod; + } + public List getQueriesFiles() { return queriesFiles; } - public void setPartialBindingType(PartialBindingType partialBindingType) { - this.partialBindingType = partialBindingType; + public void setQueriesFiles(List queriesFiles) { + this.queriesFiles = queriesFiles; } public PartialBindingType getPartialBindingType() { return partialBindingType; } + public void setPartialBindingType(PartialBindingType partialBindingType) { + this.partialBindingType = partialBindingType; + } + public boolean hasProperty(String prop){ return extraProperties.contains(prop); } @@ -357,11 +357,6 @@ public Object getProperty(String prop){ return extraProperties.getProperty(prop,null); } - public static String getConfigurationFile() { - return configurationFile; - } - - @Override public String toString() { return "Configuration{" +'\n'+ @@ -379,4 +374,9 @@ public String toString() { ", suspectsFromKG=" + suspectsFromKG +'\n'+ '}'; } + + public enum PartialBindingType {GREEDY,TEXT,NONE} + + + public enum FactsFormat {RDF, IRIS;} } diff --git a/QueryRewriting/src/main/java/extendedsldnf/EvaluatorFactory.java b/QueryRewriting/src/main/java/extendedsldnf/EvaluatorFactory.java index ce7ebd1..ed7e7e2 100755 --- a/QueryRewriting/src/main/java/extendedsldnf/EvaluatorFactory.java +++ b/QueryRewriting/src/main/java/extendedsldnf/EvaluatorFactory.java @@ -30,20 +30,27 @@ public EvaluatorFactory(Configuration config) { public RecSLDEvaluator getEvaluator(List facts, List uncertainResources,List rules) { // return getEvaluator(config.getEvaluationMethod(),facts,rules,new FactSpottingConnector(config),config.getPartialBindingType(),config.isSuspectsFromKG() ); - return getEvaluator(config.getEvaluationMethod(),facts,rules,uncertainResources,config.getPartialBindingType(),config.isSuspectsFromKG() ); + return getEvaluator(config.getEvaluationMethod(),facts,rules,uncertainResources,config.getPartialBindingType(),config.isSuspectsFromKG(), config.getMaxExplanations(), config.getMaxRuleNesting() ); } - public RecSLDEvaluator getEvaluator(Enums.EvalMethod evalMethod, List facts, List rules, List factSpottingConnectors, Configuration.PartialBindingType partialBindingType, boolean suspectsFromKG) { + + public RecSLDEvaluator getEvaluator(List facts, List uncertainResources,List rules, int maxExplanations, int maxRules) { +// return getEvaluator(config.getEvaluationMethod(),facts,rules,new FactSpottingConnector(config),config.getPartialBindingType(),config.isSuspectsFromKG() ); + return getEvaluator(config.getEvaluationMethod(),facts,rules,uncertainResources,config.getPartialBindingType(),config.isSuspectsFromKG(), maxExplanations,maxRules ); + + } + + public RecSLDEvaluator getEvaluator(Enums.EvalMethod evalMethod, List facts, List rules, List factSpottingConnectors, Configuration.PartialBindingType partialBindingType, boolean suspectsFromKG, int maxExplanations, int maxRules){ switch (evalMethod){ case SLD_ITR: - return new ItrSLDEvaluator(facts, rules,factSpottingConnectors,partialBindingType,suspectsFromKG, AbstractQueriesPool.ComparisionMethod.DFS,config.getMaxExplanations(),config.getMaxRuleNesting() ); + return new ItrSLDEvaluator(facts, rules,factSpottingConnectors,partialBindingType,suspectsFromKG, AbstractQueriesPool.ComparisionMethod.DFS,maxExplanations,maxRules); case HEURISTIC: - return new ItrSLDEvaluator(facts, rules,factSpottingConnectors,partialBindingType,suspectsFromKG, AbstractQueriesPool.ComparisionMethod.HEURISTIC,config.getMaxExplanations(),config.getMaxRuleNesting() ); + return new ItrSLDEvaluator(facts, rules,factSpottingConnectors,partialBindingType,suspectsFromKG, AbstractQueriesPool.ComparisionMethod.HEURISTIC,maxExplanations,maxRules ); case SLD: default: - return new RecSLDEvaluator(facts, rules,factSpottingConnectors,partialBindingType,suspectsFromKG,config.getMaxExplanations(),config.getMaxRuleNesting() ); + return new RecSLDEvaluator(facts, rules,factSpottingConnectors,partialBindingType,suspectsFromKG,maxExplanations,maxRules ); } diff --git a/QueryRewriting/src/main/java/extendedsldnf/RecSLDEvaluator.java b/QueryRewriting/src/main/java/extendedsldnf/RecSLDEvaluator.java index afd0648..536689b 100755 --- a/QueryRewriting/src/main/java/extendedsldnf/RecSLDEvaluator.java +++ b/QueryRewriting/src/main/java/extendedsldnf/RecSLDEvaluator.java @@ -1,25 +1,4 @@ -/* - * Integrated Rule Inference System (IRIS): - * An extensible rule inference system for datalog with extensions. - * - * Copyright (C) 2008 Semantic Technology Institute (STI) Innsbruck, - * University of Innsbruck, Technikerstrasse 21a, 6020 Innsbruck, Austria. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, - * MA 02110-1301, USA. - */ + package extendedsldnf; import config.Configuration; @@ -49,19 +28,13 @@ import static utils.Enums.ActionType.*; -//import text.ITextResult; - /** - * Implementation of the SLDNF evaluator. Please keep in mind that - * SLDNF evaluation is not capable of any tabling and can get trapped - * in an infinite loop. - * - * For details see 'Deduktive Datenbanken' by Cremers, Griefahn - * and Hinze (ISBN 978-3528047009). - * - * @author gigi + * This code follows IRIS Reasoner * + * @author Mohamed Gadelrab */ + +//TODO this class should be a template that would have subtypes. public class RecSLDEvaluator {// implements ITopDownEvaluator, IExplanationGenerator { private int maxExplanations; diff --git a/QueryRewriting/src/main/java/extendedsldnf/datastructure/InputQuery.java b/QueryRewriting/src/main/java/extendedsldnf/datastructure/InputQuery.java index da95f9e..91a212a 100755 --- a/QueryRewriting/src/main/java/extendedsldnf/datastructure/InputQuery.java +++ b/QueryRewriting/src/main/java/extendedsldnf/datastructure/InputQuery.java @@ -8,17 +8,14 @@ public class InputQuery { - private BinaryFact ntQuery; - private IQuery iQuery; int order; int label; - - + int maxRules; + int maxExplanations; List textualSources; List kgs; - - - + private BinaryFact ntQuery; + private IQuery iQuery; public InputQuery(IQuery query, int order,int label) { @@ -35,6 +32,21 @@ public InputQuery(BinaryFact ntQuery, int order,int label) { this.label=label; } + public int getMaxRules() { + return maxRules; + } + + public void setMaxRules(int maxRules) { + this.maxRules = maxRules; + } + + public int getMaxExplanations() { + return maxExplanations; + } + + public void setMaxExplanations(int maxExplanations) { + this.maxExplanations = maxExplanations; + } public IQuery getIQuery() { return iQuery; @@ -44,10 +56,18 @@ public int getOrder() { return order; } + public void setOrder(int order) { + this.order = order; + } + public int getLabel() { return label; } + public void setLabel(int label) { + this.label = label; + } + public BinaryFact getNtQuery() { return ntQuery; } @@ -64,14 +84,6 @@ public void setiQuery(IQuery iQuery) { this.iQuery = iQuery; } - public void setOrder(int order) { - this.order = order; - } - - public void setLabel(int label) { - this.label = label; - } - public List getTextualSources() { return textualSources; } diff --git a/QueryRewriting/src/main/java/extendedsldnf/datastructure/QueryExplanations.java b/QueryRewriting/src/main/java/extendedsldnf/datastructure/QueryExplanations.java index 9752735..53f617a 100755 --- a/QueryRewriting/src/main/java/extendedsldnf/datastructure/QueryExplanations.java +++ b/QueryRewriting/src/main/java/extendedsldnf/datastructure/QueryExplanations.java @@ -25,20 +25,15 @@ public class QueryExplanations implements IRelation, IQueryExplanations, SerializableData { private final double quality; + IQueryExplanations.Sorting sortingMethod=Sorting.Generation; private CostAccumulator costAccumulator; - - /** * successful rewriting paths */ private SortedSet explanations; - - @JsonAdapter(IQueryAdapter.class) private IQuery query; - IQueryExplanations.Sorting sortingMethod=Sorting.Generation; - public QueryExplanations() { this(null,new ArrayList<>(),null); @@ -117,6 +112,10 @@ public IQuery getQuery() { return this.query; } + public void setQuery(IQuery query) { + this.query = query; + } + @Override public Fact getQueryAsFact() { try { @@ -137,6 +136,10 @@ public Collection getExplanations() { return explanations; } + public void setExplanations(SortedSet explanations) { + this.explanations = explanations; + } + @Override public boolean hasDirectExplanation() { return getExplanations().stream().anyMatch(Explanation::isDirectEvidence); @@ -166,6 +169,10 @@ public CostAccumulator getCostAccumulator() { return costAccumulator; } + public void setCostAccumulator(CostAccumulator costAccumulator) { + this.costAccumulator = costAccumulator; + } + /** * count the number of documents retrieved. It sums from all explanations. However, only representative if it is a direct spotting. * @return @@ -174,18 +181,6 @@ public int documentLevelCount(){ return explanations.stream().flatMap(Explanation::getTextEvidencesStream).mapToInt(EvidenceNode::getRetrievedDocsCount).sum(); } - public void setCostAccumulator(CostAccumulator costAccumulator) { - this.costAccumulator = costAccumulator; - } - - public void setExplanations(SortedSet explanations) { - this.explanations = explanations; - } - - public void setQuery(IQuery query) { - this.query = query; - } - @Override public String getReadableString(){ StringBuilder readable=new StringBuilder("Query:\t"+getQueryAsFact().toSearchableString().replaceAll("\t"," ").trim()+"\t("+explanations.size()+")"); @@ -230,16 +225,16 @@ public Sorting getSortingMethod() { return sortingMethod; } + public void setSortingMethod(Sorting sortingMethod) { + this.sortingMethod = sortingMethod; + } + @Override public String toJsonString() { Gson gson= CustomGson.getInstance().getGson(); return gson.toJson(this); } - public void setSortingMethod(Sorting sortingMethod) { - this.sortingMethod = sortingMethod; - } - @Override public String toJSON() { return null; diff --git a/QueryRewriting/src/main/java/extendedsldnf/datastructure/TextualSource.java b/QueryRewriting/src/main/java/extendedsldnf/datastructure/TextualSource.java index 4ab99a6..75153ee 100644 --- a/QueryRewriting/src/main/java/extendedsldnf/datastructure/TextualSource.java +++ b/QueryRewriting/src/main/java/extendedsldnf/datastructure/TextualSource.java @@ -1,5 +1,7 @@ package extendedsldnf.datastructure; +import java.util.Objects; + public class TextualSource { String sourceName; @@ -9,4 +11,19 @@ public TextualSource(String sourceName, Double weight) { this.sourceName = sourceName; this.weight = weight; } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof TextualSource)) return false; + TextualSource that = (TextualSource) o; + return sourceName.equals(that.sourceName); + } + + @Override + public int hashCode() { + return Objects.hash(sourceName); + } } + diff --git a/QueryRewriting/src/main/java/text/FactSpottingConnector.java b/QueryRewriting/src/main/java/text/FactSpottingConnector.java index 065ee14..f76dd06 100755 --- a/QueryRewriting/src/main/java/text/FactSpottingConnector.java +++ b/QueryRewriting/src/main/java/text/FactSpottingConnector.java @@ -21,11 +21,17 @@ public class FactSpottingConnector implements ITextConnector{ private Logger logger = LoggerFactory.getLogger(getClass()); public FactSpottingConnector(Configuration conf) { + this(conf.getSpottingMethod(),conf); + + + } + + public FactSpottingConnector(FactSpotterFactory.SpottingMethod spottingMethod, Configuration conf) { String spottingConfFile = conf.getSpottingConfFile(); - System.out.println("Set spotting config:"+ spottingConfFile); + logger.info("Set spotting config:"+ spottingConfFile); de.mpii.factspotting.config.Configuration.setConfigurationFile(spottingConfFile==null? null:spottingConfFile,conf.getSpottingConfFileClassLoader()); - System.out.println( "Used spotting config:"+ de.mpii.factspotting.config.Configuration.getConfigurationFile()); - spotter=FactSpotterFactory.create(conf.getSpottingMethod()); + logger.info("Used spotting config:"+ de.mpii.factspotting.config.Configuration.getConfigurationFile()); + spotter=FactSpotterFactory.create(spottingMethod); } diff --git a/QueryRewriting/src/main/java/utils/DataUtils.java b/QueryRewriting/src/main/java/utils/DataUtils.java index 4073572..54f0ffc 100755 --- a/QueryRewriting/src/main/java/utils/DataUtils.java +++ b/QueryRewriting/src/main/java/utils/DataUtils.java @@ -39,6 +39,15 @@ public static IExtendedFacts loadFacts(Configuration conf,List factsFile return factsLoader.loadFacts(factsFiles); } + /** + * Creates IFacts source according to the type indicated in the configuration file + * @param conf + * @return + */ + public static IExtendedFacts loadFacts(Configuration conf,String factsFile) { + IFactsLoader factsLoader= FactsLoaderFactory.getLoader(conf); + return factsLoader.loadFacts(factsFile); + }