From 14ef98d85ed4a39379c7754ce4679a0b23bd334a Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 14 Jul 2023 14:55:27 +0200 Subject: [PATCH 1/3] trying to implement Type.match and Type.instantiate for NonTerminalType such that pattern matching works for types like {&T &E}* and &O?, such that generic functions like sort, size, zip and reverse can be written for syntax lists, such that some things become simpler for concrete syntax analysis --- src/org/rascalmpl/types/NonTerminalType.java | 22 +- src/org/rascalmpl/types/RascalType.java | 3 + .../values/parsetrees/SymbolAdapter.java | 284 ++++++++++++++++++ 3 files changed, 308 insertions(+), 1 deletion(-) diff --git a/src/org/rascalmpl/types/NonTerminalType.java b/src/org/rascalmpl/types/NonTerminalType.java index 20094a487ef..dd167251e27 100644 --- a/src/org/rascalmpl/types/NonTerminalType.java +++ b/src/org/rascalmpl/types/NonTerminalType.java @@ -39,6 +39,7 @@ import io.usethesource.vallang.ISetWriter; import io.usethesource.vallang.IValue; import io.usethesource.vallang.IValueFactory; +import io.usethesource.vallang.exceptions.FactTypeUseException; import io.usethesource.vallang.type.Type; import io.usethesource.vallang.type.TypeFactory; import io.usethesource.vallang.type.TypeFactory.RandomTypesConfig; @@ -315,7 +316,6 @@ protected Type glbWithAbstractData(Type type) { protected Type glbWithConstructor(Type type) { return TF.voidType(); } - @Override protected boolean isSupertypeOf(Type type) { @@ -325,6 +325,7 @@ protected boolean isSupertypeOf(Type type) { return super.isSupertypeOf(type); } + @Override protected boolean isSupertypeOf(RascalType type) { @@ -532,4 +533,23 @@ public IValue randomValue(Random random, IValueFactory vf, TypeStore store, Map< // TODO: this generates an on-the-fly nullable production and returns a tree for that rule return rvf.appl(vf.constructor(RascalValueFactory.Production_Default, symbol, vf.list(), vf.set()), vf.list()); } + + @Override + public Type instantiate(Map bindings) { + return RTF.nonTerminalType(SymbolAdapter.instantiate(symbol, bindings)); + } + + @Override + public boolean match(Type matched, Map bindings) throws FactTypeUseException { + matched.isSubtypeOf(this); + return isSubtypeOf(matched); + // if (matched instanceof NonTerminalType) { + // return SymbolAdapter.match(symbol, ((NonTerminalType) matched).symbol, bindings); + // } + // else { + // IRascalValueFactory vf = IRascalValueFactory.getInstance(); + + // return SymbolAdapter.match(symbol, matched.asSymbol(vf, new TypeStore(), vf.setWriter(), new HashSet<>()), bindings); + // } + } } diff --git a/src/org/rascalmpl/types/RascalType.java b/src/org/rascalmpl/types/RascalType.java index 2dac9fd2983..463d067a3f8 100644 --- a/src/org/rascalmpl/types/RascalType.java +++ b/src/org/rascalmpl/types/RascalType.java @@ -1,5 +1,8 @@ package org.rascalmpl.types; +import java.util.Map; + +import io.usethesource.vallang.exceptions.FactTypeUseException; import io.usethesource.vallang.type.ExternalType; import io.usethesource.vallang.type.Type; diff --git a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java index 991d6ed459c..1e5c102dc15 100644 --- a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java @@ -62,7 +62,11 @@ import static org.rascalmpl.values.RascalValueFactory.Symbol_Value; import static org.rascalmpl.values.RascalValueFactory.Symbol_Void; +import java.util.Map; + import org.rascalmpl.exceptions.ImplementationError; +import org.rascalmpl.types.NonTerminalType; +import org.rascalmpl.types.RascalTypeFactory; import org.rascalmpl.values.RascalValueFactory; import org.rascalmpl.values.ValueFactoryFactory; @@ -73,6 +77,9 @@ import io.usethesource.vallang.IString; import io.usethesource.vallang.IValue; import io.usethesource.vallang.IValueFactory; +import io.usethesource.vallang.type.Type; +import io.usethesource.vallang.type.TypeFactory; +import io.usethesource.vallang.type.TypeStore; public class SymbolAdapter { private static final IValueFactory VF = ValueFactoryFactory.getValueFactory(); @@ -1084,4 +1091,281 @@ private static int rangeEnd(IConstructor range) { public static boolean isParametrizableType(IConstructor sort) { return SymbolAdapter.isADT(sort) || SymbolAdapter.isParameterizedSort(sort) || SymbolAdapter.isParameterizedLex(sort); } + + public static IConstructor instantiate(IConstructor symbol, Map bindings) { + // This could also be written in Rascal. After a nice bootstrap, it would be so much + // shorter + if (isLabel(symbol) || isOpt(symbol) || isSet(symbol) || isList(symbol) || isBag(symbol) || isReifiedType(symbol) || isConditional(symbol)) { + return symbol.set("symbol", instantiate(getSymbol(symbol), bindings)); + } + + if (isIterPlusSeps(symbol) || isIterStarSeps(symbol)) { + IList seps = getSeparators(symbol); + IConstructor elem = getSymbol(symbol); + + return symbol + .set("symbol", instantiate(elem, bindings)) + .set("separators", seps.stream().map(s -> instantiate((IConstructor) s, bindings)).collect(VF.listWriter())); + } + + if (isIterPlus(symbol) || isIterStar(symbol)) { + IConstructor elem = getSymbol(symbol); + + return symbol.set("symbol", instantiate(elem, bindings)); + } + + if (isSeq(symbol) || isRel(symbol) || isListRel(symbol) || isTuple(symbol)) { + IList symbols = getSymbols(symbol); + + return symbol.set("symbols", symbols.stream().map(s -> instantiate((IConstructor) s, bindings)).collect(VF.listWriter())); + } + + if (isAlt(symbol)) { + ISet alts = getAlternatives(symbol); + + return symbol.set("alternatives", alts.stream().map(a -> instantiate((IConstructor) a, bindings)).collect(VF.setWriter())); + } + + if (isParameterizedSort(symbol) || isParameterizedLex(symbol) || isADT(symbol) || isAlias(symbol)) { + IList params = (IList) symbol.get("parameters"); + + return symbol.set("parameters", params.stream().map(s -> instantiate((IConstructor) s, bindings)).collect(VF.listWriter())); + } + + if (isStartSort(symbol)) { + return symbol.set("symbol", instantiate(getStart(symbol), bindings)); + } + + if (isParameter(symbol)) { + String name = getName(symbol); + Type parameterType = TypeFactory.getInstance().parameterType(name); + Type resolved = bindings.get(parameterType); + + if (resolved != null) { + if (resolved instanceof NonTerminalType) { + return ((NonTerminalType) resolved).getSymbol(); + } + else { + return TypeFactory.getInstance().asSymbol(resolved, VF, new TypeStore(), VF.setWriter()); + } + } + + // if we can not find it, it remains open. + return symbol; + } + + if (isMap(symbol)) { + return symbol + .set("from", instantiate((IConstructor) symbol.get("from"), bindings)) + .set("to", instantiate((IConstructor) symbol.get("to"), bindings)); + } + + if (isFunc(symbol)) { + IList parameters = (IList) symbol.get("parameters"); + return symbol + .set("ret", instantiate((IConstructor) symbol.get("ret"), bindings)) + .set("parameters", parameters.stream().map(s -> instantiate((IConstructor) s, bindings)).collect(VF.listWriter())); + } + + if (isCons(symbol)) { + IList parameters = (IList) symbol.get("parameters"); + + return symbol + .set("ret", instantiate((IConstructor) symbol.get("ret"), bindings)) + .set("parameters", parameters.stream().map(s -> instantiate((IConstructor) s, bindings)).collect(VF.listWriter())); + } + + // all the other symbols are not composed of other symbols + // sort, lex, keyword, empty, char-class, layout, literal, ci-lit, int, str, real, node, value, num, datetime and loc + + return symbol; + } + + /* + * This implements a type-match algorithm similar to {@see Type.match()}. It binds type parameters + * as a side-effect when the pattern matches and returns true iff the pattern and the subject + * are the same kind of type. + */ + public static boolean match(IConstructor symbol, IConstructor subject, Map bindings) { + symbol = stripLabelsAndConditions(symbol); + subject = stripLabelsAndConditions(subject); + + // fast path equality is fine, but no more binding can be learned from that either + // this happens a lot because types are not accidentally constructed from each other. + if (isEqual(symbol, subject)) { + return true; + } + + if (isLayouts(symbol) && isLayouts(subject)) { + return true; + } + + if (isIterPlusSeps(symbol) || isIterStarSeps(symbol)) { + if (isIterPlusSeps(subject) || isIterStarSeps(subject)) { + // both are separated + IList seps1 = getSeparators(symbol); + IList seps2 = getSeparators(subject); + IConstructor elem1 = getSymbol(symbol); + IConstructor elem2 = getSymbol(subject); + + // here we deal with the presence/absence of layout in the pattern and the subject + IConstructor sep1 = (IConstructor) (seps1.size() == 3 ? seps1.get(1) : seps1.get(0)); + IConstructor sep2 = (IConstructor) (seps2.size() == 3 ? seps2.get(1) : seps2.get(0)); + + return match(elem1, elem2, bindings) + && match(sep1, sep2, bindings); + } + else if (isIterPlus(subject) || isIterStar(subject)) { + // the subject is not separated yet, probably the other one only by layout? + // TODO: check the layout separator + IConstructor elem1 = getSymbol(symbol); + IConstructor elem2 = getSymbol(subject); + return match(elem1, elem2, bindings); + } + else { + return false; + } + } + + if (isIterPlus(symbol) || isIterStar(symbol)) { + if (isIterPlusSeps(subject) || isIterStarSeps(subject)) { + // the subject is separated but the patter not (yet) + // TODO check if the subject only has layout + IConstructor elem1 = getSymbol(symbol); + IConstructor elem2 = getSymbol(subject); + + return match(elem1, elem2, bindings); + } + else if (isIterPlus(subject) || isIterStar(subject)) { + // both are not separated + IConstructor elem1 = getSymbol(symbol); + IConstructor elem2 = getSymbol(subject); + return match(elem1, elem2, bindings); + } + else { + return false; + } + } + + // this happens when we match against abstract patterns with concrete trees + if (isADT(symbol) && (isSort(subject) || isLex(subject))) { + return getName(symbol).equals(getName(subject)); + } + + // after this we can assume we are looking at similar structures + if (symbol.getConstructorType() != subject.getConstructorType()) { + return false; + } + + if (isOpt(symbol) || isSet(symbol) || isList(symbol) || isBag(symbol) || isReifiedType(symbol) || isConditional(symbol)) { + return match(getSymbol(symbol), getSymbol(subject), bindings); + } + + + + if (isSeq(symbol) || isRel(symbol) || isListRel(symbol) || isTuple(symbol)) { + IList symbols1 = getSymbols(symbol); + IList symbols2 = getSymbols(subject); + + if (symbols1.size() != symbols2.size()) { + return false; + } + + var is2 = symbols2.iterator(); + + return symbols1.stream() + .map(s1 -> match((IConstructor) s1, (IConstructor) is2.next(), bindings)) + .reduce(true, (a,b) -> a && b); + } + + if (isAlt(symbol)) { + ISet alts1 = getAlternatives(symbol); + ISet alts2 = getAlternatives(subject); + + var is2 = alts2.iterator(); + + // TODO: this does not backtrack for different orders if the match fails + return alts1.stream() + .map(s1 -> match((IConstructor) s1, (IConstructor) is2.next(), bindings)) + .reduce(true, (a,b) -> a && b); + } + + if (isParameterizedSort(symbol) || isParameterizedLex(symbol) || isADT(symbol) || isAlias(symbol)) { + IList params1 = (IList) symbol.get("parameters"); + IList params2 = (IList) symbol.get("parameters"); + + if (params1.size() != params2.size()) { + return false; + } + + var is2 = params2.iterator(); + + return params1.stream() + .map(s1 -> match((IConstructor) s1, (IConstructor) is2.next(), bindings)) + .reduce(true, (a,b) -> a && b); + } + + if (isStartSort(symbol)) { + return match(getStart(symbol), getStart(subject), bindings); + } + + if (isParameter(symbol)) { + String name = getName(symbol); + Type parameterType = TypeFactory.getInstance().parameterType(name); + Type resolved = bindings.get(parameterType); + + // if it is already bound, lub with that, otherwise lub with void + if (resolved == null) { + resolved = TypeFactory.getInstance().voidType(); + } + + resolved = resolved.lub(RascalTypeFactory.getInstance().nonTerminalType(subject)); + + bindings.put(parameterType, resolved); + + return true; + } + + if (isMap(symbol)) { + return match((IConstructor) symbol.get("from"), (IConstructor) subject.get("from"), bindings) + && match((IConstructor) symbol.get("to"), (IConstructor) subject.get("to"), bindings); + } + + if (isFunc(symbol)) { + IList parameters1 = (IList) symbol.get("parameters"); + IList parameters2 = (IList) subject.get("parameters"); + + if (parameters1.size() != parameters2.size()) { + return false; + } + + var is2 = parameters2.iterator(); + + return match((IConstructor) symbol.get("ret"), (IConstructor) subject.get("ret"), bindings) + && parameters1.stream() + .map(s1 -> match((IConstructor) s1, (IConstructor) is2.next(), bindings)) + .reduce(true, (a,b) -> a && b); + } + + if (isCons(symbol)) { + IList parameters1 = (IList) symbol.get("parameters"); + IList parameters2 = (IList) subject.get("parameters"); + + if (parameters1.size() != parameters2.size()) { + return false; + } + + var is2 = parameters2.iterator(); + + return match((IConstructor) symbol.get("ret"), (IConstructor) subject.get("ret"), bindings) + && parameters1.stream() + .map(s1 -> match((IConstructor) s1, (IConstructor) is2.next(), bindings)) + .reduce(true, (a,b) -> a && b); + } + + // all the other symbols are not composed of other symbols + // sort, lex, keyword, empty, char-class, layout, literal, ci-lit, int, str, real, node, value, num, datetime and loc + + return symbol.equals(subject); + } } From 084fa32cc1968a7510e73a4bd0e41d214ecc2932 Mon Sep 17 00:00:00 2001 From: "Jurgen J. Vinju" Date: Fri, 14 Jul 2023 15:33:02 +0200 Subject: [PATCH 2/3] fixed a bug wrt void matching against non-terminal types, which happens regularly when matching empty lists of parse trees --- src/org/rascalmpl/types/NonTerminalType.java | 21 ++++++++------- .../values/parsetrees/SymbolAdapter.java | 26 +++++++------------ 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/src/org/rascalmpl/types/NonTerminalType.java b/src/org/rascalmpl/types/NonTerminalType.java index dd167251e27..9c35deae569 100644 --- a/src/org/rascalmpl/types/NonTerminalType.java +++ b/src/org/rascalmpl/types/NonTerminalType.java @@ -541,15 +541,18 @@ public Type instantiate(Map bindings) { @Override public boolean match(Type matched, Map bindings) throws FactTypeUseException { - matched.isSubtypeOf(this); - return isSubtypeOf(matched); - // if (matched instanceof NonTerminalType) { - // return SymbolAdapter.match(symbol, ((NonTerminalType) matched).symbol, bindings); - // } - // else { - // IRascalValueFactory vf = IRascalValueFactory.getInstance(); + // return matched.isSubtypeOf(this); + if (matched.isSubtypeOf(TypeFactory.getInstance().voidType())) { + return true; + + } + else if (matched instanceof NonTerminalType) { + return SymbolAdapter.match(symbol, ((NonTerminalType) matched).symbol, bindings); + } + else { + IRascalValueFactory vf = IRascalValueFactory.getInstance(); - // return SymbolAdapter.match(symbol, matched.asSymbol(vf, new TypeStore(), vf.setWriter(), new HashSet<>()), bindings); - // } + return SymbolAdapter.match(symbol, matched.asSymbol(vf, new TypeStore(), vf.setWriter(), new HashSet<>()), bindings); + } } } diff --git a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java index 1e5c102dc15..8ce65f214b7 100644 --- a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java @@ -1190,6 +1190,10 @@ public static boolean match(IConstructor symbol, IConstructor subject, Map Date: Fri, 14 Jul 2023 16:22:30 +0200 Subject: [PATCH 3/3] fixed more issues --- src/org/rascalmpl/types/NonTerminalType.java | 11 ++ .../values/parsetrees/SymbolAdapter.java | 108 +++++++++++++++--- 2 files changed, 102 insertions(+), 17 deletions(-) diff --git a/src/org/rascalmpl/types/NonTerminalType.java b/src/org/rascalmpl/types/NonTerminalType.java index 9c35deae569..c14546056b8 100644 --- a/src/org/rascalmpl/types/NonTerminalType.java +++ b/src/org/rascalmpl/types/NonTerminalType.java @@ -543,16 +543,27 @@ public Type instantiate(Map bindings) { public boolean match(Type matched, Map bindings) throws FactTypeUseException { // return matched.isSubtypeOf(this); if (matched.isSubtypeOf(TypeFactory.getInstance().voidType())) { + // this is a common fast path otherwise handled by the else case down here return true; } else if (matched instanceof NonTerminalType) { + // we have to implement this on the symbol level since we do not have separate types + // for every different kind of non-terminal (regular symbols, literals, etc) return SymbolAdapter.match(symbol, ((NonTerminalType) matched).symbol, bindings); } else { IRascalValueFactory vf = IRascalValueFactory.getInstance(); + // here we lift the other types to symbols, such that they can be compared. + // this is mainly necessary for the different kinds of parametrized sorts and + // how they match against the ADT names and parameters. return SymbolAdapter.match(symbol, matched.asSymbol(vf, new TypeStore(), vf.setWriter(), new HashSet<>()), bindings); } } + + @Override + public boolean isOpen() { + return SymbolAdapter.isOpen(symbol); + } } diff --git a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java index 8ce65f214b7..4b7ed14d37f 100644 --- a/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java +++ b/src/org/rascalmpl/values/parsetrees/SymbolAdapter.java @@ -1193,7 +1193,7 @@ public static boolean match(IConstructor symbol, IConstructor subject, Map isOpen((IConstructor) s)).reduce(false, (a,b) -> a || b); + } + + if (isIterPlus(symbol) || isIterStar(symbol)) { + IConstructor elem = getSymbol(symbol); + + return isOpen(elem); + } + + if (isSeq(symbol) || isRel(symbol) || isListRel(symbol) || isTuple(symbol)) { + IList symbols = getSymbols(symbol); + + + return symbols.stream().map(s -> isOpen((IConstructor) s)).reduce(false, (a,b) -> a || b); + } + + if (isAlt(symbol)) { + ISet alts = getAlternatives(symbol); + + return alts.stream().map(s -> isOpen((IConstructor) s)).reduce(false, (a,b) -> a || b); + } + + if (isParameterizedSort(symbol) || isParameterizedLex(symbol) || isADT(symbol) || isAlias(symbol)) { + IList params = (IList) symbol.get("parameters"); + + return params.stream().map(s -> isOpen((IConstructor) s)).reduce(false, (a,b) -> a || b); + } + + if (isStartSort(symbol)) { + return isOpen(getStart(symbol)); + } + + if (isParameter(symbol)) { + return true; + } + + if (isMap(symbol)) { + return + isOpen((IConstructor) symbol.get("from")) + || isOpen((IConstructor) symbol.get("to")); + } + + if (isFunc(symbol)) { + IList parameters = (IList) symbol.get("parameters"); + return isOpen((IConstructor) symbol.get("ret")) + || parameters.stream().map(s -> isOpen((IConstructor) s)).reduce(false, (a,b) -> a || b); + } + + if (isCons(symbol)) { + IList parameters = (IList) symbol.get("parameters"); + + return isOpen((IConstructor) symbol.get("ret")) + || parameters.stream().map(s -> isOpen((IConstructor) s)).reduce(false, (a,b) -> a || b); + } + + // all the other symbols are not composed of other symbols + // sort, lex, keyword, empty, char-class, layout, literal, ci-lit, int, str, real, node, value, num, datetime and loc + + return false; + } }