Skip to content

Commit

Permalink
adding convection to code
Browse files Browse the repository at this point in the history
  • Loading branch information
@gadelrab
gadelrab committed Jul 10, 2016
1 parent 6685c65 commit 9d25c61
Show file tree
Hide file tree
Showing 7 changed files with 148 additions and 186 deletions.
17 changes: 13 additions & 4 deletions src/main/java/de/mpii/frequentrulesminning/AssociationRuleMiningSPMF.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ public AssocRulesExtended getFrequentAssociationRules(String inputRDFFile, Strin
}


revisetedRuleQuality(rules,transactionsDB);
revisedRuleQuality(rules,transactionsDB);

long estimatedTime = System.nanoTime() - startTime;
System.out.println("Done! ----------------------------------------------------- estimatedTime= "+estimatedTime);
Expand Down Expand Up @@ -224,14 +224,16 @@ public void ruleAtTimeExceptionRanking(TransactionsDatabase transactionsDB, Asso

}

private void revisetedRuleQuality(AssocRulesExtended rules, TransactionsDatabase transactionsDB) {
private void revisedRuleQuality(AssocRulesExtended rules, TransactionsDatabase transactionsDB) {
Evaluator eval=new Evaluator(transactionsDB);
rules.getRules().parallelStream().forEach((r)->{
final ExceptionItem e=r.getTopException();
r.setRevisedConfidence(e==null? r.getConfidence():eval.confidence(r,e));
r.setRevisedLift(e==null? r.getLift():eval.lift(r,e));
r.setRevisedJaccardCoefficient(e==null? r.getJaccardCoefficient():eval.JaccardCoefficient(r,e));
r.setRevisedBodyCoverage(e==null? r.getBodyCoverage():eval.bodyCoverage(r,e));
r.setRevisedConviction(e==null? r.getConviction():eval.conviction(r,e));

});

}
Expand Down Expand Up @@ -267,6 +269,7 @@ private void rulesQuality(AssocRulesExtended rules, TransactionsDatabase transac
r.setNegConfidence(eval.negativeRuleConfidence(r));
r.setJaccardCoefficient(eval.JaccardCoefficient(r));
r.setBodyCoverage(eval.bodyCoverage(r));
r.setConviction(eval.conviction(r));

});

Expand Down Expand Up @@ -651,7 +654,7 @@ public void showStatisticsRevisedRules(AssocRulesExtended rules,boolean export,



st.append("topK\ttype\tAvgConfRO\tdiff\tAvgLIFTRO\tdiff\tAvgJaccardCofRO\tdiff\tBodyCoverageRO\tdiff");
st.append("topK\ttype\tAvgConfRO\tdiff\tAvgLIFTRO\tdiff\tAvgJaccardCofRO\tdiff\tBodyCoverageRO\tdiff\tConvictionRO\tdiff");
st.append('\n');

StringBuilder stAll=new StringBuilder();
Expand All @@ -666,13 +669,15 @@ public void showStatisticsRevisedRules(AssocRulesExtended rules,boolean export,
double orgAvgLiftRO = rules.getRevisedRulesLiftStats(k, false).getAverage();
double orgAvgJaccardCoefficientRO = rules.getRevisedRulesJaccardCoefficientStats(k, false).getAverage();
double orgAvgBodyCoverageRO = rules.getRevisedRulesBodyCoverageStats(k, false).getAverage();
double orgAvgConvRO= rules.getRevisedRulesConvictionStats(k, false).getAverage();


st.append(k+"\tBefore\t");
st.append(String.format("%.6f",orgAvgConfRO)+"\t__\t");
st.append(String.format("%.6f", orgAvgLiftRO)+"\t__\t");
st.append(String.format("%.5f", orgAvgJaccardCoefficientRO)+"\t__\t");
st.append(String.format("%.6f", orgAvgBodyCoverageRO) + "\t__\t");
st.append(String.format("%.6f", orgAvgConvRO) + "\t__\t");
st.append('\n');


Expand All @@ -682,7 +687,7 @@ public void showStatisticsRevisedRules(AssocRulesExtended rules,boolean export,
double newAvgJaccardCoefficientRO = rules.getRevisedRulesJaccardCoefficientStats(k, true).getAverage();;
double newAvgLiftRO = rules.getRevisedRulesLiftStats(k, true).getAverage();
double newAvgBodyCoverageRO = rules.getRevisedRulesBodyCoverageStats(k, true).getAverage();
;
double newAvgConvRO = rules.getRevisedRulesConvictionStats(k, true).getAverage();

st.append(k+"\tAfter\t");
st.append(String.format("%.6f", newAvgConfidenceRO)+"\t");
Expand All @@ -693,6 +698,8 @@ public void showStatisticsRevisedRules(AssocRulesExtended rules,boolean export,
st.append(String.format("%.6f", newAvgJaccardCoefficientRO-orgAvgJaccardCoefficientRO)+"\t");
st.append(String.format("%.6f", newAvgBodyCoverageRO) + "\t");
st.append(String.format("%.6f", orgAvgBodyCoverageRO-newAvgBodyCoverageRO)+"\t");
st.append(String.format("%.6f", newAvgConvRO)+"\t");
st.append(String.format("%.6f", newAvgConvRO-orgAvgConvRO)+"\t");

st.append('\n');

Expand Down Expand Up @@ -721,6 +728,8 @@ public void showStatisticsRevisedRules(AssocRulesExtended rules,boolean export,
stAll.append("\tBodyCov\t");
stAll.append(rules.getRevisedRulesBodyCoverageDiffStats(k).toString().replaceAll("DoubleSummaryStatistics","")+"\n");



// int[] numArray=rules.getRules().stream().filter(AssocRuleWithExceptions::hasExceptions).mapToInt(AssocRuleWithExceptions::getExceptionCandidatesSize).toArray();
// Arrays.sort(numArray);
// double median;
Expand Down
247 changes: 68 additions & 179 deletions src/main/java/de/mpii/frequentrulesminning/Evaluator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,11 @@ public double lift(AssocRuleWithExceptions rule, ExceptionItem exceptionItem) {
public double lift(AssocRuleWithExceptions rule, Set<Transaction> ruleTransactions, Set<Transaction> bodyTransactions, Set<Transaction> headTransactions, ExceptionItem exceptionItem) {
double ruleSupport = TransactionsDatabase.getTransactionsCount(ruleTransactions, rule.getBodyAndHead(), ExceptionItem.toArray(exceptionItem), this.useWeights);
double bodySupport = TransactionsDatabase.getTransactionsCount(bodyTransactions, rule.getBody(), ExceptionItem.toArray(exceptionItem), this.useWeights);
double headSupport = TransactionsDatabase.getTransactionsCount(headTransactions, rule.getHead(), ExceptionItem.toArray(exceptionItem), this.useWeights);


// TODO: we do not want to capture exceptions in head
// double headSupport = TransactionsDatabase.getTransactionsCount(headTransactions, rule.getHead(), ExceptionItem.toArray(exceptionItem), this.useWeights);
double headSupport = TransactionsDatabase.getTransactionsCount(headTransactions, rule.getHead(), null, this.useWeights);


return computeLift(ruleSupport, bodySupport, headSupport);
Expand Down Expand Up @@ -273,30 +277,6 @@ public void setUseWeights(boolean useWeights) {
this.useWeights = useWeights;
}

// public void setUseOrder(boolean useOrder) {
// this.useOrder = useOrder;
// }

// /**
// * Computes the average confidence of (head <-body, not exceptionItem) and (not head <- body, exceptionItem)
// *
// * @param rule
// * @param exceptionItem
// */
// public double computePosNegConfidence(AssocRuleWithExceptions rule, ExceptionItem exceptionItem) {
//
// double positiveConf = this.confidence(rule, exceptionItem);
//
// double negConf = negativeRuleConfidence(rule, exceptionItem);
//
// return computePosNegConfidence(positiveConf,negConf);
//
//
// }
//
// public static double computePosNegConfidence(double positiveConf,double negConf){
// return (positiveConf + negConf) / 2;
// }


public double negativeRuleConfidence(AssocRuleWithExceptions rule) {
Expand Down Expand Up @@ -373,172 +353,81 @@ public double bodyCoverage(AssocRuleWithExceptions rule, ExceptionItem exception
}


//TODO Commented as they do not handle weights
public double conviction(AssocRuleWithExceptions rule){
return conviction(null);
}

// /**
// * Coverage of set of rules R with same head using coverage = (1/E) * ((supp(r1)* supp(r2)..sup(rn))^(1/|R|) )
// *
// * @param head
// * @param rules
// * @param withExceptions
// */
// public void groupCoverage(HeadGroup head, Collection<AssocRuleWithExceptions> rules, boolean withExceptions) {
//
// //TODO Exception Handling is not yet implemented
// Set<Transaction> containsBodyOrHead = transactionsDB.getTransactions(head.getHeadItemsIds(), null, false);
//
// //ArrayList<Integer> rulesTransactionsCount=new ArrayList<>();
// BigDecimal coverageMultiplication = BigDecimal.ONE;
// for (AssocRuleWithExceptions rule : rules) {
//
//
// Set<Transaction> bodyTransactions = rule.getBodyTransactions();
// double bodyTransactionsCount = TransactionsDatabase.getTransactionsCount(bodyTransactions);
//
// if (bodyTransactionsCount == 0 || coverageMultiplication.equals(BigDecimal.ZERO))
// System.out.println(rule + " " + bodyTransactions.size() + " " + bodyTransactionsCount + " " + coverageMultiplication);
// coverageMultiplication.multiply(BigDecimal.valueOf(bodyTransactionsCount));
//
//
// // combine to all bodies transactions
//
// containsBodyOrHead = Sets.union(containsBodyOrHead, bodyTransactions);
//
//

public double conviction(AssocRuleWithExceptions rule, ExceptionItem exceptionItem){
Set<Transaction> ruleTransactions = transactionsDB.getTransactions(rule.getBodyAndHead(), ExceptionItem.toArray(exceptionItem), this.countPrediction);
Set<Transaction> bodyTransactions = transactionsDB.getTransactions(rule.getBody(), ExceptionItem.toArray(exceptionItem), this.countPrediction);
Set<Transaction> headTransactions = transactionsDB.getTransactions(rule.getHead(), null, this.countPrediction);

// if (this.useOrder) {
// ruleTransactions = TransactionsDatabase.filterBetterQualityRulesPredictions(ruleTransactions, rule);
// bodyTransactions = TransactionsDatabase.filterBetterQualityRulesPredictions(bodyTransactions, rule);
// headTransactions = TransactionsDatabase.filterBetterQualityRulesPredictions(headTransactions, rule);
// }
//
//
// //int coverageMultiplication=rulesTransactionsCount.stream().reduce((i,j)-> i*j).get();
// double allTransactionsCount = TransactionsDatabase.getTransactionsCount(containsBodyOrHead);
// int rSize = rules.size();
//
// double groupCoverage = Math.pow((coverageMultiplication.doubleValue()), (1.0D / (double) rSize)) / ((double) allTransactionsCount);
//// double groupCoverage= coverageMultiplication.pow((1.0D/(double)rSize))/((double)allTransactionsCount);
// if (groupCoverage == 0 || allTransactionsCount == 0)
// System.out.print("groupCoverage = " + groupCoverage + " coverageMultiplication = " + coverageMultiplication + " rSize: = " + rSize + " all Transactions Count = " + allTransactionsCount);
// head.setCoverage(groupCoverage);
//// head.setAllTransactionsCount(allTransactionsCount);
// }

ruleTransactions = filterTransactions(ruleTransactions, rule);
bodyTransactions = filterTransactions(bodyTransactions, rule);
headTransactions =filterTransactions(headTransactions, rule);

// public void groupCoverage(HeadGroup head, Collection<AssocRuleWithExceptions> assocRuleWithExceptionses) {
// groupCoverage(head, assocRuleWithExceptionses, false);
//
// }
return conviction(rule, ruleTransactions, bodyTransactions, headTransactions, exceptionItem);

// public double groupConfidence(HeadGroup head, Collection<AssocRuleWithExceptions> assocRuleWithExceptionses) {
// return groupConfidence(head, assocRuleWithExceptionses, false);
// }
}

public double conviction(AssocRuleWithExceptions rule, Set<Transaction> ruleTransactions, Set<Transaction> bodyTransactions, Set<Transaction> headTransactions, ExceptionItem exceptionItem) {

// /**
// * Measures a set of rule confidence by computing the support of all rules union over the union of the bodies supp(all rules)/supp(bodies)
// *
// * @param head
// * @param rules
// * @param withExceptions
// */
// public double groupConfidence(HeadGroup head, Collection<AssocRuleWithExceptions> rules, boolean withExceptions) {
//
// Set<Transaction> containsBody = new HashSet<>();
//
//
// for (AssocRuleWithExceptions rule : rules) {
//// Set<Transaction> bodyTransactions = transactionsDB.getTransactions(rule.getItemset1(), null);
// Set<Transaction> bodyTransactions = rule.getBodyTransactions();
// //containsBody.addAll(bodyTransactions);
// containsBody = Sets.union(containsBody, bodyTransactions);
// }
//
// double bodyTransactionsCount = TransactionsDatabase.getTransactionsCount(containsBody);
//
// double rulesBodyandHeadCount = TransactionsDatabase.getTransactionsCount(transactionsDB.filterTransactionsWith(containsBody, head.getHeadItemsIds(), false));
//
// //head.setConfidence((double) rulesBodyandHeadCount / bodyTransactionsCount);
// return (double) rulesBodyandHeadCount / bodyTransactionsCount;
//
// }
double ruleSupport = TransactionsDatabase.getTransactionsCount(ruleTransactions, rule.getBodyAndHead(), ExceptionItem.toArray(exceptionItem), this.useWeights);
double bodySupport = TransactionsDatabase.getTransactionsCount(bodyTransactions, rule.getBody(), ExceptionItem.toArray(exceptionItem), this.useWeights);
double headSupport = TransactionsDatabase.getTransactionsCount(headTransactions, rule.getHead(), null, this.useWeights);

return computeConviction(ruleSupport,bodySupport,headSupport, transactionsDB.getSize());

// public void exceptionsConflictScore(AssocRuleWithExceptions targetRule, Set<AssocRuleWithExceptions> groupRules, AssocRulesExtended rulesSource) {
// // compute intersection score for each exception
// for (ExceptionItem e : targetRule.getExceptionCandidates()) {
// // TODO compute score for each exception
// double conflictScore = 0;
// int conflictTransactionsCount = 0;
// for (AssocRuleWithExceptions rule : groupRules) {
// if (targetRule.equals(rule)) {
// continue;
// }
//
// // filter transactions that contains the body and the excpetion
//// Set<Transaction> transactionsWithBodyandException=transactionsDB.filterTransactionsWith(predictableTransactions.get(rule), ArrayUtils.addAll(e.getItems(),targetRule.getBody()));
// Set<Transaction> transactionsWithBodyandException = transactionsDB.filterTransactionsWith(rule.getSafePredictableTransactions(), ArrayUtils.addAll(e.getItems(), targetRule.getBody()), false);
// double singleConflictTransactionsCount = TransactionsDatabase.getTransactionsCount(transactionsWithBodyandException);
// conflictScore += (singleConflictTransactionsCount * rule.getConfidence());
// conflictTransactionsCount += singleConflictTransactionsCount;
//
//
// }
// e.setConflictScore(conflictTransactionsCount == 0 ? 0 : (conflictScore / conflictTransactionsCount));
// e.setConflictCount(conflictTransactionsCount);
//
// // ComputeInverted conflict
// if (rulesSource != null)
// invertedConflictScore(targetRule, e, rulesSource.getRules(e.getItems(), null));
//
// }
// }


// public void conflict(HeadGroup group, AssocRulesExtended rulesSource) {
//
// Set<AssocRuleWithExceptions> groupRules = group.getRules();
// for (AssocRuleWithExceptions rule : groupRules) {
//// exceptionsConflictScore(rule,predictableTransactions,rulesSource);
// exceptionsConflictScore(rule, groupRules, rulesSource);
// }
//
// }
}

private double computeConviction(double ruleSupport, double bodySupport, double headSupport, double kbSize) {

double confidence=computeConfidence(ruleSupport,bodySupport);

return ((kbSize-headSupport)/kbSize)/(1-confidence);

}


public double negativeRuleConviction(AssocRuleWithExceptions rule) {
return negativeRuleConviction(rule,null);
}

public double negativeRuleConviction(AssocRuleWithExceptions rule, ExceptionItem exceptionItem){


Set<Transaction> bodyWithExceptionTransactions = transactionsDB.getTransactions(ArrayUtils.addAll(rule.getBody(), ExceptionItem.toArray(exceptionItem)), null, this.countPrediction);
Set<Transaction> ruleTransactions = transactionsDB.getTransactions(ArrayUtils.addAll(rule.getBody(), ExceptionItem.toArray(exceptionItem)), rule.getHead(), this.countPrediction);
//TODO check again this.countPrediction can cause problems in case of switching it on in filterOutTransactionsWith(transactions, withoutItems, 0, false/*!withPredictions*/);
Set<Transaction> notHeadTransactions= transactionsDB.getTransactions(null, rule.getHead(), this.countPrediction);


ruleTransactions = filterTransactions(ruleTransactions, rule);
bodyWithExceptionTransactions = filterTransactions(bodyWithExceptionTransactions, rule);
//TODO error: when removing less quality rules predictions we will loose some transactions
notHeadTransactions= filterTransactions(notHeadTransactions, rule);

return negativeRuleConviction(rule, bodyWithExceptionTransactions, notHeadTransactions,ruleTransactions, exceptionItem);

}

private double negativeRuleConviction(AssocRuleWithExceptions rule, Set<Transaction> bodyWithExceptionTransactions, Set<Transaction> notHeadTransactions, Set<Transaction> ruleTransactions, ExceptionItem exceptionItem) {
double ruleSupport = TransactionsDatabase.getTransactionsCount(ruleTransactions, ArrayUtils.addAll(rule.getBody(), ExceptionItem.toArray(exceptionItem)), rule.getHead(), this.useWeights);
double bodySupport = TransactionsDatabase.getTransactionsCount(bodyWithExceptionTransactions, ArrayUtils.addAll(rule.getBody(), ExceptionItem.toArray(exceptionItem)), null, this.useWeights);
double headSupport = TransactionsDatabase.getTransactionsCount(notHeadTransactions,null,rule.getHead() , this.useWeights);

return computeConviction(ruleSupport,bodySupport,headSupport, transactionsDB.getSize());

}

// /**
// * Computes the conflict score between the Exception Candidate generated from rules and positive examples.
// *
// * @param targetRule
// * @param exceptionCandidate
// * @param exceptionRules
// * @return
// */
// public void invertedConflictScore(AssocRuleWithExceptions targetRule, ExceptionItem exceptionCandidate, Set<AssocRuleWithExceptions> exceptionRules) {
//
//
// // positive examples of the rule that fo not contain the target exception
// Set<Transaction> ruleTransactionsWithoutException = transactionsDB.filterOutTransactionsWith(targetRule.getHornRuleTransactions(), exceptionCandidate.getItems(), false);
//
//// HashMap<AssocRuleWithExceptions,Set<Transaction>> predictableTransactions=new HashMap<>(exceptionRules.size());
// double conflictScore = 0;
// int conflictTransactionsCount = 0;
// for (AssocRuleWithExceptions rule : exceptionRules) {
// if (!targetRule.isBodySubsetOf(rule))
// continue;
//
// // transactions with the body but neither the exceptions nor the head ... they are predictable with this rule
//// Set<Transaction> rulePredictableTransactions=rule.getPredicatableTransactions(transactionsDB,true);
// Set<Transaction> rulePredictableTransactions = rule.getSafePredictableTransactions();
//// predictableTransactions.put (rule, rulePredictableTransactions);
//
// // Count the intersection between the exception predection and the target rule positive examples
// double predicatableExceptionCount = TransactionsDatabase.getTransactionsCount(Sets.intersection(rulePredictableTransactions, ruleTransactionsWithoutException));
// conflictScore += (predicatableExceptionCount * rule.getConfidence());
// conflictTransactionsCount += predicatableExceptionCount;
//
// }
//
//// return conflictTransactionsCount==0? 0:(conflictScore/conflictTransactionsCount);
// exceptionCandidate.setInvertedConflictScore(conflictTransactionsCount == 0 ? 0 : (conflictScore / conflictTransactionsCount));
// exceptionCandidate.setInvertedConflictCount(conflictTransactionsCount);
//
// }

}
Loading

0 comments on commit 9d25c61

Please sign in to comment.