Compute image of whitespace tokens lazily

Nobody cares and they make up a significant part of
token sequences. Alternatively we could intern them
which would most likely yield near 100% cache hit,
because of indentation patterns.
This commit is contained in:
Clément Fournier
2019-08-06 09:42:48 +02:00
parent 1339ea888d
commit b3860a3d57
4 changed files with 26 additions and 6 deletions

View File

@ -67,7 +67,7 @@ public class JavaccToken implements GenericToken, java.io.Serializable {
private final CharSequence image;
private final int startInclusive;
private final int endExclusive;
private final TokenDocument document;
protected final TokenDocument document;
/** {@link #undefined()} */
private JavaccToken() {

View File

@ -104,11 +104,6 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ro
return classTypeResolver;
}
@Override
public ASTCompilationUnit getRoot() {
return this;
}
@InternalApi
@Deprecated
public void setClassTypeResolver(ClassTypeResolver classTypeResolver) {

View File

@ -85,6 +85,7 @@ public interface JavaNode extends ScopedNode, TextAvailableNode {
@Override
JavaNode jjtGetParent();
GenericToken jjtGetFirstToken();

View File

@ -32,6 +32,18 @@ final class JavaTokenUtils {
jcs.getEndOffset(),
jcs.getTokenDocument()
);
case JavaParserConstants.WHITESPACE:
// unlikely that anybody cares about that, and since
// they're still 30% of all tokens best make this assumption
// btw 40% of all tokens have a compile-time string constant
// as image (jjstrLiteralImages) so they're shared.
return new LazyImageToken(
kind,
jcs.getStartOffset(),
jcs.getEndOffset(),
jcs.getTokenDocument()
);
default:
return new JavaccToken(
kind,
@ -47,6 +59,18 @@ final class JavaTokenUtils {
return token instanceof GTToken ? ((GTToken) token).realKind : token.kind;
}
private static final class LazyImageToken extends JavaccToken {
public LazyImageToken(int kind, int startInclusive, int endExclusive, TokenDocument document) {
super(kind, null, startInclusive, endExclusive, document);
}
@Override
public String getImage() {
return document.getFullText().substring(getStartInDocument(), getEndInDocument());
}
}
private static final class GTToken extends JavaccToken {
final int realKind;