From e76a1d6eb8d32f620a567f76e4a8713a2d57a503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 09:21:48 +0200 Subject: [PATCH] Aha! fix tests of c++ --- .../ast/impl/javacc/JavaccTokenDocument.java | 72 ++--------- .../util/document/FragmentedDocBuilder.java | 2 +- .../util/document/FragmentedTextDocument.java | 20 ++- .../pmd/util/document/RootTextDocument.java | 2 +- .../pmd/util/document/TextDocument.java | 6 +- .../impl/javacc/io/JavaEscapeReaderTest.java | 118 +++--------------- .../sourceforge/pmd/cpd/CppBlockSkipper.java | 10 +- .../cpd/testdata/continuation_intra_token.txt | 7 +- 8 files changed, 58 insertions(+), 179 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index b37537ac7b..201720662b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -22,13 +22,14 @@ import net.sourceforge.pmd.util.document.TextDocument; */ public class JavaccTokenDocument extends TokenDocument { + private final TokenDocumentBehavior behavior; final StringPool stringPool = new StringPool(); private JavaccToken first; - private TextDocument translatedDocument; public JavaccTokenDocument(TextDocument textDocument, TokenDocumentBehavior behavior) { super(textDocument); + this.behavior = behavior; } public static class TokenDocumentBehavior { @@ -70,7 +71,7 @@ public class JavaccTokenDocument extends TokenDocument { * @see EscapeTranslator */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { - return text; + return translator.apply(text); } @@ -149,7 +150,7 @@ public class JavaccTokenDocument extends TokenDocument { * {@link CharStream#appendSuffix(StringBuilder, int)} a noop. */ public boolean useMarkSuffix() { - return false; + return behavior.useMarkSuffix(); } /** @@ -161,19 +162,7 @@ public class JavaccTokenDocument extends TokenDocument { * @see EscapeTranslator */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { - return text; - } - - @Override - public TextDocument getTextDocument() { - return translatedDocument == null ? super.getTextDocument() - : translatedDocument; - } - - final void doTranslate() throws MalformedSourceException { - if (translatedDocument == null) { - translatedDocument = translate(super.getTextDocument()); - } + return behavior.translate(text); } /** @@ -213,7 +202,7 @@ public class JavaccTokenDocument extends TokenDocument { } protected boolean isImagePooled(JavaccToken t) { - return false; + return behavior.isImagePooled(t); } /** @@ -224,55 +213,14 @@ public class JavaccTokenDocument extends TokenDocument { * @return A descriptive string */ public final @NonNull String describeKind(int kind) { - if (kind == JavaccToken.IMPLICIT_TOKEN) { - return ""; - } - String impl = describeKindImpl(kind); - if (impl != null) { - return impl; - } - return ""; + return behavior.describeKind(kind); } /** - * Describe the given kind. If this returns a non-null value, then - * that's what {@link #describeKind(int)} will use. Otherwise a default - * implementation is used. - * - *

An implementation typically uses the JavaCC-generated array - * named {@code Constants.tokenImage}. Remember to - * check the bounds of the array. - * - * @param kind Kind of token - * - * @return A descriptive string, or null to use default - */ - protected @Nullable String describeKindImpl(int kind) { - return null; - } - - - /** - * Creates a new token with the given kind. This is called back to - * by JavaCC-generated token managers (jjFillToken). Note that a - * created token is not guaranteed to end up in the final token chain. - * - * @param kind Kind of the token - * @param cs Char stream of the file. This can be used to get text - * coordinates and the image - * @param image Shared instance of the image token. If this is non-null, - * then no call to {@link CharStream#getTokenImage()} should be - * issued. - * - * @return A new token + * @see TokenDocumentBehavior#createToken(JavaccTokenDocument, int, CharStream, String) */ public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { - return new JavaccToken( - kind, - image == null ? cs.getTokenImageCs() : image, - cs.getStartOffset(), - cs.getEndOffset(), - this - ); + return behavior.createToken(this, kind, cs, image); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java index aec408bef1..4813435bec 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java @@ -75,7 +75,7 @@ public final class FragmentedDocBuilder { } public int inputOffsetAt(int outputOffset) { - return FragmentedTextDocument.inputOffsetAt(outputOffset, firstFragment); + return FragmentedTextDocument.inputOffsetAt(outputOffset, firstFragment, true); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java index add5706d51..f04338996e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java @@ -38,10 +38,10 @@ final class FragmentedTextDocument implements TextDocument { } @Override - public int inputOffset(int outputOffset) { + public int inputOffset(int outputOffset, boolean inclusive) { // todo this would be pretty slow when we're in the middle of some escapes // we could check save the fragment last accessed to speed it up, and look forwards & backwards - return base.inputOffset(inputOffsetAt(outputOffset, firstFragment)); + return base.inputOffset(inputOffsetAt(outputOffset, firstFragment, inclusive), inclusive); } @Override @@ -86,8 +86,8 @@ final class FragmentedTextDocument implements TextDocument { @Override public @NonNull TextRegion inputRegion(TextRegion outputRegion) { - return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset()), - inputOffset(outputRegion.getEndOffset())); + return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true), + inputOffset(outputRegion.getEndOffset(), false)); } @Override @@ -95,14 +95,22 @@ final class FragmentedTextDocument implements TextDocument { base.close(); } - static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment) { + static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment, boolean inclusive) { Fragment f = firstFragment; if (f == null) { return outputOffset; } - while (f.next != null && f.inEnd() < outputOffset) { + while (f.next != null && f.outEnd() < outputOffset) { f = f.next; } + if (!inclusive && f.outEnd() == outputOffset) { + if (f.next != null) { + f = f.next; + // fallthrough + } else { + return f.outToIn(outputOffset) + 1; + } + } return f.outToIn(outputOffset); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index 42d123b3d4..aa1c10ba63 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -112,7 +112,7 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { } @Override - public int inputOffset(int outOffset) { + public int inputOffset(int outOffset, boolean inclusive) { return outOffset; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 591a0fd51e..f07a7870a1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -94,16 +94,18 @@ public interface TextDocument extends Closeable { * * * @param outOffset Output offset + * @param inclusive Whether the offset is to be interpreted as the index of a character (true), + * or the position after a character (false) * * @return Input offset */ - int inputOffset(int outOffset); + int inputOffset(int outOffset, boolean inclusive); /** * Translate a region given in the the coordinate system of this * document, to the coordinate system of the original document. * This works as if creating a new region with both start and end - * offsets translated through {@link #inputOffset(int)}. The + * offsets translated through {@link #inputOffset(int, boolean)}. The * returned region may have a different length. * * @param outputRegion Output region diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index b6922c8a23..a72f7e760c 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -6,11 +6,9 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; -import net.sourceforge.pmd.lang.DummyLanguageModule; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -18,9 +16,11 @@ import net.sourceforge.pmd.util.document.TextDocument; public class JavaEscapeReaderTest { - @NonNull - public JavaEscapeTranslator readString(String input) { - return new JavaEscapeTranslator(TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); + public TextDocument readString(String input) { + TextDocument intext = TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); + try (JavaEscapeTranslator translator = new JavaEscapeTranslator(intext)) { + return translator.translateDocument(); + } } @@ -28,14 +28,8 @@ public class JavaEscapeReaderTest { public void testSimpleRead() throws IOException { String input = "abcdede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, chars.length); - - Assert.assertEquals("Should have read the entire text", input.length(), read); - Assert.assertEquals(input, new String(chars, 0, input.length())); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap(input),r.getText()); } } @@ -43,14 +37,8 @@ public class JavaEscapeReaderTest { public void testNotAnEscape1Read() throws IOException { String input = "abc\\dede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, chars.length); - - Assert.assertEquals("Should have read the entire text", input.length(), read); - Assert.assertEquals(input, new String(chars, 0, input.length())); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap(input),r.getText()); } } @@ -58,40 +46,8 @@ public class JavaEscapeReaderTest { public void testNotAnEscape1Read2() throws IOException { String input = "abc\\\\\\dede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, chars.length); - - Assert.assertEquals("Should have read the entire text", input.length(), read); - Assert.assertEquals(input, new String(chars, 0, input.length())); - } - } - - @Test - public void testNotAnEscape1Read3SplitInTheMiddleOfBackslashes() throws IOException { - - String input = "abc\\\\\\dede"; - // ^ - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, 4); - - Assert.assertEquals("Should have read just enough", 4, read); - assertBufferIsJust("abc\\", chars, 0); - - read = r.read(chars, 4, 1); - - Assert.assertEquals(1, read); - assertBufferIsJust("abc\\\\", chars, 0); - - read = r.read(chars, 5, chars.length - 5); - - Assert.assertEquals(5, read); - assertBufferIsJust("abc\\\\\\dede", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap(input),r.getText()); } } @@ -99,19 +55,8 @@ public class JavaEscapeReaderTest { public void testAnEscapeStopAtEnd() throws IOException { String input = "abc\\\\\\u00a0dede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, 4); - - Assert.assertEquals(4, read); - assertBufferIsJust("abc\u00a0", chars, 0); - - read = r.read(chars, 4, 2); - - Assert.assertEquals(2, read); - assertBufferIsJust("abc\u00a0de", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap("abc\u00a0dede"), r.getText()); } } @@ -119,19 +64,8 @@ public class JavaEscapeReaderTest { public void testSeveralEscapes() throws IOException { String input = "abc\\\\\\u00a0d\\uu00a0ede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[20]; - - int read = r.read(chars, 0, 5); - - Assert.assertEquals(5, read); - assertBufferIsJust("abc\u00a0d", chars, 0); - - read = r.read(chars, 5, 4); - - Assert.assertEquals(4, read); - assertBufferIsJust("abc\u00a0d\u00a0ede", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap("abc\u00a0d\u00a0ede"), r.getText()); } } @@ -139,26 +73,8 @@ public class JavaEscapeReaderTest { public void testAnEscapeInsideBlock() throws IOException { String input = "abc\\\\\\u00a0dede\\u00a0"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, 12); - - Assert.assertEquals(9, read); - assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); - - read = r.read(chars, 9, chars.length - 9); - - Assert.assertEquals(-1, read); - assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap("abc\u00a0dede\u00a0"), r.getText()); } } - - private static void assertBufferIsJust(String contents, char[] chars, int off) { - // asserts the rest of the buffer is null characters - char[] chars2 = new char[chars.length]; - contents.getChars(0, contents.length(), chars2, off); - Assert.assertArrayEquals(chars2, chars); - } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java index 694d5cd978..fa5be8ee91 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java @@ -18,19 +18,23 @@ import net.sourceforge.pmd.util.document.TextDocument; class CppBlockSkipper extends EscapeTranslator { private final Pattern skipStart; + private final String skipStartMarker; private final Pattern skipEnd; + private final String skipEndMarker; public CppBlockSkipper(TextDocument original, String skipStartMarker, String skipEndMarker) { super(original); - skipStart = Pattern.compile("^" + Pattern.quote(skipStartMarker)); - skipEnd = Pattern.compile("^" + Pattern.quote(skipEndMarker)); + skipStart = Pattern.compile("^(?i)" + Pattern.quote(skipStartMarker), Pattern.MULTILINE); + this.skipStartMarker = "\n" + skipStartMarker; + skipEnd = Pattern.compile("^(?i)" + Pattern.quote(skipEndMarker), Pattern.MULTILINE); + this.skipEndMarker = "\n" + skipEndMarker; } @Override protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException { Matcher start = skipStart.matcher(input).region(this.bufpos, maxOff); if (start.find()) { - Matcher end = skipStart.matcher(input).region(start.end(), maxOff); + Matcher end = skipEnd.matcher(input).region(start.end(), maxOff); if (end.find()) { return recordEscape(start.start(), end.end(), Chars.EMPTY); } diff --git a/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt b/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt index abc147ce72..febb035697 100644 --- a/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt +++ b/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt @@ -1,10 +1,11 @@ [Image] or [Truncated image[ Bcol Ecol L1 - [void] 1 1 + [void] 1 2 L5 - [main] 2 1 + [main] 2 2 +L9 + [(] 2 2 L10 - [(] 1 2 [)] 2 2 L12 [{] 2 2