From 0d5270a2dcbf6dc7746b45d50357ee4be9bfa479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 05:56:23 +0200 Subject: [PATCH 001/136] Checkout javacc utils --- .../sourceforge/pmd/lang/ast/CharStream.java | 61 +++--- .../lang/ast/impl/io/EscapeAwareReader.java | 155 +++++++++++++++ .../pmd/lang/ast/impl/io/EscapeTracker.java | 188 ++++++++++++++++++ .../pmd/lang/ast/impl/io/JavaInputReader.java | 134 +++++++++++++ .../pmd/lang/ast/impl/io/LineTracker.java | 21 ++ .../pmd/lang/ast/impl/io/NewCharStream.java | 78 ++++++++ 6 files changed, 611 insertions(+), 26 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java index 5cf6044ae7..4565dafc42 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java @@ -5,6 +5,7 @@ package net.sourceforge.pmd.lang.ast; +import java.io.EOFException; import java.io.IOException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; @@ -13,8 +14,8 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; * PMD flavour of character streams used by JavaCC parsers. * * TODO for when all JavaCC languages are aligned: - * * rename methods to match decent naming conventions - * * move to impl.javacc package + * * rename methods to match decent naming conventions + * * move to impl.javacc package */ public interface CharStream { @@ -24,7 +25,8 @@ public interface CharStream { * * @return The next character * - * @throws IOException If the underlying char stream throws + * @throws EOFException Upon EOF + * @throws IOException If the underlying char stream throws EOF */ char readChar() throws IOException; @@ -35,14 +37,14 @@ public interface CharStream { * the buffer between two successive calls to this method to implement * backup correctly. */ - char BeginToken() throws IOException; // SUPPRESS CHECKSTYLE we'll rename it later + char BeginToken() throws IOException; /** * Returns a string made up of characters from the token mark up to * to the current buffer position. */ - String GetImage(); // SUPPRESS CHECKSTYLE we'll rename it later + String GetImage(); /** @@ -53,7 +55,7 @@ public interface CharStream { * * 
{@code
      * String t = tokenImage();
-     * return t.substring(t.length() - len, t.length()).toCharArray();
+     * return t.substring(t.length() - len).toCharArray();
      * }
* * @param len Length of the returned array @@ -63,7 +65,16 @@ public interface CharStream { * @throws IndexOutOfBoundsException If len is greater than the length of the * current token */ - char[] GetSuffix(int len); // SUPPRESS CHECKSTYLE we'll rename it later + default char[] GetSuffix(int len) { + String t = GetImage(); + return t.substring(t.length() - len).toCharArray(); + } + + + default void appendSuffix(StringBuilder sb, int len) { + String t = GetImage(); + sb.append(t, t.length() - len, t.length()); + } /** @@ -80,12 +91,6 @@ public interface CharStream { */ void backup(int amount); - @Deprecated - int getBeginColumn(); - - @Deprecated - int getBeginLine(); - /** Returns the column number of the last character for the current token. */ int getEndColumn(); @@ -94,7 +99,23 @@ public interface CharStream { /** Returns the line number of the last character for current token. */ int getEndLine(); - // These methods are added by PMD + + default int getBeginColumn() { + return -1; + } + + + default int getBeginLine() { + return -1; + } + + + /** Returns the start offset of the current token (in the original source), inclusive. */ + int getStartOffset(); + + + /** Returns the end offset of the current token (in the original source), exclusive. */ + int getEndOffset(); /** @@ -105,16 +126,4 @@ public interface CharStream { return null; // for VelocityCharStream } - - /** Returns the start offset of the current token (in the original source), inclusive. */ - default int getStartOffset() { - return -1; - } - - - /** Returns the end offset of the current token (in the original source), exclusive. */ - default int getEndOffset() { - return -1; - } - } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java new file mode 100644 index 0000000000..e026544557 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java @@ -0,0 +1,155 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.io; + +import static java.lang.Integer.min; + +import java.io.IOException; +import java.io.Reader; +import java.nio.CharBuffer; + +import net.sourceforge.pmd.util.StringUtil; + +/** + * A reader that optionally escapes its input text. It records where + * escapes occurred, and can translate an offset in the translated + * input document to a line+column position in the original input. + * + *

The default implementation does not perform any escaping. + */ +@SuppressWarnings("PMD.AssignmentInOperand") +public class EscapeAwareReader extends Reader { + + /** + * Source characters. When there is an escape, eg \ u00a0, the + * first backslash is replaced with the translated value of the + * escape. The bufpos is updated so that we read the next char + * after the escape. + * + *

This makes it so that 1. we don't need an additional buffer for + * translated chars, and 2. the full escape is preserved, just use + * the {@link EscapeTracker} to get the position of the escape and + * replace the first char with a backslash. We can report unnecessary + * escapes that way. + */ + protected char[] input; + /** Position of the next char to read in the input. */ + protected int bufpos; + /** Keep track of adjustments to make to the offsets, caused by unicode escapes. */ + final EscapeTracker escapes = new EscapeTracker(); + + public EscapeAwareReader(CharSequence input, int startIdxInclusive, int endIdxExclusive) { + assert input != null; + assert startIdxInclusive >= 0; + assert endIdxExclusive >= 0; + assert endIdxExclusive >= startIdxInclusive; + + int len = endIdxExclusive - startIdxInclusive; + + this.input = new char[len]; + input.toString().getChars(startIdxInclusive, endIdxExclusive, this.input, 0); + bufpos = 0; + } + + public EscapeAwareReader(CharSequence input) { + this(input, 0, input.length()); + } + + /** + * Translate all the characters in the buffer. + */ + public int translate() throws IOException { + return read(null, 0, Integer.MAX_VALUE); + } + + + @Override + public int read(final char[] cbuf, final int off, final int len) throws IOException { + ensureOpen(); + if (this.bufpos == input.length) { + return -1; + } + + int readChars = 0; + while (readChars < len && this.bufpos < input.length) { + int bpos = this.bufpos; + int nextJump = gobbleMaxWithoutEscape(bpos, len - readChars); + int newlyReadChars = nextJump - bpos; + + assert newlyReadChars >= 0 && (readChars + newlyReadChars) <= len; + + if (newlyReadChars != 0) { + if (cbuf != null) { + System.arraycopy(input, bpos, cbuf, off + readChars, newlyReadChars); + } + } else if (nextJump == input.length) { + // eof + break; + } + readChars += newlyReadChars; + } + return readChars; + } + + + /** + * Returns the max offset, EXclusive, with which we can cut the input + * array from the bufpos to dump it into the output array. This sets + * the bufpos to where we should start the next jump. + */ + protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { + return this.bufpos = min(bufpos + maxReadahead, input.length); + } + + protected void recordEscape(final int startOffsetInclusive, int length) { + this.escapes.recordEscape(startOffsetInclusive, length); + } + + @Override + public void close() throws IOException { + this.bufpos = -1; + this.input = null; + } + + + /** Check to make sure that the stream has not been closed */ + protected void ensureOpen() throws IOException { + if (input == null) { + throw new IOException("Stream closed"); + } + } + + @Override + public boolean ready() throws IOException { + ensureOpen(); + return true; + } + + /** + * Returns the offset in the input text of the given translated offset. + * This includes the length of any unicode escapes. + * + * 

+     * input:      "a\u00a0b"
+     * translated: "a b"
+     *
+     * inputOffset(0) = 0
+     * inputOffset(1) = 1
+     * inputOffset(2) = 7 // includes the length of the escape
+     *
+ */ + public int inputOffset(int outputOffset) { + return escapes.inputOffsetAt(outputOffset); + } + + public int getLine(int idxInInput) { + return StringUtil.lineNumberAt(CharBuffer.wrap(input), idxInInput); + } + + public int getColumn(int idxInInput) { + return StringUtil.columnNumberAt(CharBuffer.wrap(input), idxInInput); + } + +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java new file mode 100644 index 0000000000..8e39d172d3 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java @@ -0,0 +1,188 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.io; + +/** + * Records where escapes occurred in the input document. This is quite + * an inefficient way to deal with it, yet in the common case where there + * are few/no escapes, it's enough I think. + */ +class EscapeTracker { + + private static final int[] EMPTY = new int[0]; + + /** + * Offsets in the input buffer where a unicode escape occurred. + * Represented as pairs [off, len] where + * - off is the offset in the source file where the escape occurred + * - len is the length in characters of the escape (which is translated to a single char). + */ + private int[] escapeRecords = EMPTY; + /** Index of the next write in the {@link #escapeRecords}. */ + private int nextFreeIdx = 0; + + /** + * Calls to this method must occur in source order (ie param + * offsetInInput increases monotonically). + */ + void recordEscape(int offsetInInput, int len) { + if (nextFreeIdx + 1 >= escapeRecords.length) { + // double capacity, add 1 to not stay stuck at zero + int[] newOffsets = new int[(escapeRecords.length + 1) * 2]; + System.arraycopy(escapeRecords, 0, newOffsets, 0, escapeRecords.length); + this.escapeRecords = newOffsets; + } + + escapeRecords[nextFreeIdx++] = offsetInInput; + escapeRecords[nextFreeIdx++] = len - 1; // -1 because the translated escape has length 1 + } + + /** + * Convert an offset in the translated file into an offset in + * the untranslated input. + */ + public int inputOffsetAt(int translatedOffset) { + // basically accumulate the lengths of all escapes occurring before the given translatedOffset + int sum = translatedOffset; + for (int i = 0; i < nextFreeIdx; i += 2) { + if (escapeRecords[i] < sum) { + sum += escapeRecords[i + 1]; + } else { + break; + } + } + return sum; + } + + @Override + public String toString() { + StringBuilder res = new StringBuilder("Escape set {"); + for (int i = 0; i < nextFreeIdx; i += 2) { + res.append("(at=").append(escapeRecords[i]).append(", len=").append(escapeRecords[i + 1]).append("), "); + } + + return res.append('}').toString(); + } + + class Cursor { + + + /** + * This is the index in buf of the next char to read, it always + * holds that buf[pos] is a valid character. + */ + private int pos; + + /** + * Index in {@link #escapeRecords} of the next escape that occurred + * in the source (this means, the buffer is discontinuous at + * inputOffsets[nextEscape]) + */ + private int nextEscape = 0; + + /** + * This is the current offset in the translated document, it + * is shifted from pos by the total length of the escapes that + * occurred before pos. + */ + private int outOffset = 0; + + /** + * A char buffer, which has discontinuities at the indexes + * identified by the {@link #escapeRecords}. It must hold + * that buf.length is the original source length. + */ + private final char[] buf; + + private int mark = Integer.MAX_VALUE; + private int markEscape; + private int markOutOffset; + + Cursor(char[] buf) { + this.buf = buf; + } + + char next() { + char c = buf[pos]; + + if (nextEscape < escapeRecords.length && pos == escapeRecords[nextEscape]) { + pos += escapeRecords[nextEscape + 1]; // add escape length + this.nextEscape += 2; + } else { + pos++; + } + outOffset++; + return c; + } + + void backup(int numChars) { + outOffset -= numChars; + + if (nextEscape <= 0) { + pos -= numChars; // then there were no escapes before the 'pos' + } else { + int off = pos; + for (int i = nextEscape - 2; i >= 0 && numChars > 0; i -= 2) { + int esc = escapeRecords[i]; + if (esc == off) { + off -= escapeRecords[i + 1]; + } else if (esc > off) { + // then the current escape was before what we're looking at + break; + } else { + off--; + } + numChars--; + } + pos = off - numChars; + } + } + + void mark() { + this.mark = pos; + this.markEscape = nextEscape; + this.markOutOffset = outOffset; + } + + void markToString(StringBuilder sb) { + assert mark <= pos : "Wrong mark"; + assert markEscape <= nextEscape : "Wrong mark"; + + int prevLength = sb.length(); + + if (markEscape == nextEscape) { + // no escape in the marked range + sb.append(buf, mark, pos); + } else { + sb.ensureCapacity(markLength()); + + int cur = mark; + int esc = markEscape; + while (cur < pos && esc < nextEscape) { + int escapeOff = escapeRecords[esc]; + assert escapeOff < pos; + sb.append(buf, cur, escapeOff + 1); + cur = escapeOff + escapeRecords[esc + 1]; + esc += 2; + } + // no more escape in the range, append everything until the pos + sb.append(buf, cur, pos + 1); + assert sb.length() - prevLength == markLength(); + } + } + + int curOutOffset() { + return outOffset; + } + + int markOutOffset() { + return markOutOffset; + } + + int markLength() { + return outOffset - markOutOffset; + } + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java new file mode 100644 index 0000000000..a3bbcd4dfa --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java @@ -0,0 +1,134 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.io; + +import static java.lang.Integer.min; + +import java.io.BufferedReader; +import java.io.IOException; + +/** + * An implementation of java.io.Reader that translates Java unicode escapes. + * This implementation has efficient block IO but poor char-by-char performance. + * If this is required, wrap it into a {@link BufferedReader}. + */ +@SuppressWarnings("PMD.AssignmentInOperand") +public final class JavaInputReader extends EscapeAwareReader { + + /** + * An offset until which we read backslashes and decided they were not + * an escape. The read procedure may cut off in the middle of the escape, + * and turn an even num of backslashes into an odd one, so until we crossed + * this offset, backslashes are not treated specially. + */ + private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; + + public JavaInputReader(CharSequence input, int startIdxInclusive, int endIdxExclusive) { + super(input, startIdxInclusive, endIdxExclusive); + } + + public JavaInputReader(CharSequence input) { + super(input); + } + + + /** + * Returns the max offset, EXclusive, with which we can cut the input + * array from the bufpos to dump it into the output array. This sets + * the bufpos to where we should start the next jump. + */ + @Override + protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { + int off = bufpos; + int max = min(bufpos + maxReadahead, input.length); + boolean noBackSlash = false; + int notEscapeEnd = this.savedNotEscapeSpecialEnd; + while (off < max && (noBackSlash = input[off] != '\\' || notEscapeEnd < off)) { + off++; + } + + if (noBackSlash) { + this.bufpos = off; + return off; + } + + final int firstBslashOff = off; + while (off < input.length && input[off] == '\\') { + off++; + } + + int bslashCount = off - firstBslashOff; + // this condition is "is there an escape at offset firstBslashOff" + if ((bslashCount & 1) == 1 // odd number of backslashes + && off < input.length - 4 // at least 5 chars to form the escape ('u' + 4 hex digits) + && input[off] == 'u') { // the char after the last backslash is a 'u' + + replaceFirstBackslashWithEscape(firstBslashOff, off); + this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; + this.bufpos = off + 5; + this.recordEscape(firstBslashOff, off + 5 - firstBslashOff); + return firstBslashOff + 1; + } else { + // not an escape sequence + int min = min(bufpos + maxReadahead, off); + // save the number of backslashes that are part of the escape, + // might have been cut in half by the maxReadahead + this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; + this.bufpos = min; + return min; + } + } + + private void replaceFirstBackslashWithEscape(int posOfFirstBackSlash, int offOfTheU) throws IOException { + try { + char c = (char) + ( hexVal(input[++offOfTheU]) << 12 + | hexVal(input[++offOfTheU]) << 8 + | hexVal(input[++offOfTheU]) << 4 + | hexVal(input[++offOfTheU]) + ); + input[posOfFirstBackSlash] = c; // replace the start char of the backslash + } catch (NumberFormatException e) { + + String message = "Invalid escape sequence at line " + + getLine(posOfFirstBackSlash) + ", column " + + getColumn(posOfFirstBackSlash); + + throw new IOException(message, e); + } + } + + private static int hexVal(char c) { + switch (c) { + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + return c - '0'; + case 'A': + case 'B': + case 'C': + case 'D': + case 'E': + case 'F': + return c - ('A' - 10); + case 'a': + case 'b': + case 'c': + case 'd': + case 'e': + case 'f': + return c - ('a' - 10); + default: + throw new NumberFormatException("Character '" + c + "' is not a valid hexadecimal digit"); + } + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java new file mode 100644 index 0000000000..b7442125cd --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java @@ -0,0 +1,21 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.io; + + +class LineTracker { + + private int[] lineOffsets = new int[200]; + private int cur; + + void recordLine(int offset) { + if (cur == lineOffsets.length) { + int[] newOffsets = new int[this.lineOffsets.length + 200]; + System.arraycopy(lineOffsets, 0, newOffsets, 0, lineOffsets.length); + this.lineOffsets = newOffsets; + } + lineOffsets[cur++] = offset; + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java new file mode 100644 index 0000000000..18fa601b90 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java @@ -0,0 +1,78 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.io; + +import java.io.IOException; + +import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeTracker.Cursor; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; + +public class NewCharStream implements CharStream { + + private final JavaccTokenDocument document; + private final Cursor cursor; + + public NewCharStream(JavaccTokenDocument document, EscapeTracker.Cursor cursor) { + this.document = document; + this.cursor = cursor; + } + + public static CharStream consume(EscapeAwareReader reader, JavaccTokenDocument doc) throws IOException { + try (EscapeAwareReader r = reader) { + reader.translate(); + return new NewCharStream(doc, reader.escapes.new Cursor(reader.input)); + } + } + + @Override + public JavaccTokenDocument getTokenDocument() { + return document; + } + + @Override + public char readChar() { + return cursor.next(); + } + + @Override + public char BeginToken() { + char c = cursor.next(); + cursor.mark(); + return c; + } + + @Override + public String GetImage() { + StringBuilder sb = new StringBuilder(); + cursor.markToString(sb); + return sb.toString(); + } + + @Override + public void backup(int amount) { + cursor.backup(amount); + } + + @Override + public int getEndColumn() { + return 0; // TODO + } + + @Override + public int getEndLine() { + return 0; // TODO + } + + @Override + public int getStartOffset() { + return cursor.markOutOffset(); + } + + @Override + public int getEndOffset() { + return cursor.curOutOffset(); + } +} From b31edc52c284757c53081200a1241ee6582c366b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 06:34:20 +0200 Subject: [PATCH 002/136] Add Chars --- .../pmd/lang/ast/impl/io/LineTracker.java | 21 ------------------- 1 file changed, 21 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java deleted file mode 100644 index b7442125cd..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/LineTracker.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.io; - - -class LineTracker { - - private int[] lineOffsets = new int[200]; - private int cur; - - void recordLine(int offset) { - if (cur == lineOffsets.length) { - int[] newOffsets = new int[this.lineOffsets.length + 200]; - System.arraycopy(lineOffsets, 0, newOffsets, 0, lineOffsets.length); - this.lineOffsets = newOffsets; - } - lineOffsets[cur++] = offset; - } -} From b779abd08b500a5f94ff592f5e72bbfc9de4e252 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 07:25:22 +0200 Subject: [PATCH 003/136] Align cpd --- .../pmd/cpd/internal/JavaCCTokenizer.java | 18 +-- .../lang/ast/impl/io/EscapeAwareReader.java | 28 ++--- .../pmd/lang/ast/impl/io/EscapeTracker.java | 8 +- .../pmd/lang/ast/impl/io/JavaInputReader.java | 29 ++--- .../pmd/lang/ast/impl/io/NewCharStream.java | 8 +- .../ast/impl/javacc/CharStreamFactory.java | 69 ---------- .../lang/ast/impl/javacc/JavaCharStream.java | 119 ------------------ .../ast/impl/javacc/JavaccTokenDocument.java | 13 ++ .../ast/impl/javacc/SimpleCharStream.java | 20 --- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 45 ++++--- .../pmd/lang/cpp/ast/CppCharStream.java | 80 ------------ .../pmd/lang/cpp/ast/CppEscapeReader.java | 61 +++++++++ .../cpp/ast => cpd}/CppCharStreamTest.java | 21 +++- .../sourceforge/pmd/cpd/JavaTokenizer.java | 8 +- .../pmd/lang/java/ast/JavaParser.java | 6 - .../pmd/lang/java/ast/JavaTokenDocument.java | 8 ++ .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 19 ++- .../pmd/lang/plsql/ast/PLSQLParser.java | 1 - .../sourceforge/pmd/cpd/PythonTokenizer.java | 6 +- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 19 ++- 20 files changed, 198 insertions(+), 388 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamFactory.java delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaCharStream.java delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/SimpleCharStream.java delete mode 100644 pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java create mode 100644 pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java rename pmd-cpp/src/test/java/net/sourceforge/pmd/{lang/cpp/ast => cpd}/CppCharStreamTest.java (61%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 4c4d248aaf..5e3c06a735 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -5,9 +5,6 @@ package net.sourceforge.pmd.cpd.internal; import java.io.IOException; -import java.io.Reader; - -import org.apache.commons.io.input.CharSequenceReader; import net.sourceforge.pmd.cpd.SourceCode; import net.sourceforge.pmd.cpd.TokenEntry; @@ -18,20 +15,23 @@ import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory; +import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; -import net.sourceforge.pmd.util.IOUtil; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.util.document.TextDocument; +import net.sourceforge.pmd.util.document.io.TextFile; public abstract class JavaCCTokenizer implements Tokenizer { @SuppressWarnings("PMD.CloseResource") protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { - Reader reader = IOUtil.skipBOM(new CharSequenceReader(sourceCode.getCodeBuffer())); - return makeLexerImpl(makeCharStream(reader)); + TextDocument textDocument = TextDocument.create(TextFile.cpdCompat(sourceCode)); + JavaccTokenDocument tokenDoc = newTokenDoc(textDocument); + return makeLexerImpl(NewCharStream.open(tokenDoc)); } - protected CharStream makeCharStream(Reader sourceCode) throws IOException { - return CharStreamFactory.simpleCharStream(sourceCode); + protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + return new JavaccTokenDocument(textDoc); } protected abstract TokenManager makeLexerImpl(CharStream sourceCode); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java index e026544557..69ee25d9d7 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java @@ -11,6 +11,7 @@ import java.io.Reader; import java.nio.CharBuffer; import net.sourceforge.pmd.util.StringUtil; +import net.sourceforge.pmd.util.document.Chars; /** * A reader that optionally escapes its input text. It records where @@ -34,29 +35,18 @@ public class EscapeAwareReader extends Reader { * replace the first char with a backslash. We can report unnecessary * escapes that way. */ - protected char[] input; + protected Chars input; /** Position of the next char to read in the input. */ protected int bufpos; /** Keep track of adjustments to make to the offsets, caused by unicode escapes. */ final EscapeTracker escapes = new EscapeTracker(); - public EscapeAwareReader(CharSequence input, int startIdxInclusive, int endIdxExclusive) { + public EscapeAwareReader(Chars input) { assert input != null; - assert startIdxInclusive >= 0; - assert endIdxExclusive >= 0; - assert endIdxExclusive >= startIdxInclusive; - - int len = endIdxExclusive - startIdxInclusive; - - this.input = new char[len]; - input.toString().getChars(startIdxInclusive, endIdxExclusive, this.input, 0); + this.input = input.mutableCopy(); bufpos = 0; } - public EscapeAwareReader(CharSequence input) { - this(input, 0, input.length()); - } - /** * Translate all the characters in the buffer. */ @@ -68,12 +58,12 @@ public class EscapeAwareReader extends Reader { @Override public int read(final char[] cbuf, final int off, final int len) throws IOException { ensureOpen(); - if (this.bufpos == input.length) { + if (this.bufpos == input.length()) { return -1; } int readChars = 0; - while (readChars < len && this.bufpos < input.length) { + while (readChars < len && this.bufpos < input.length()) { int bpos = this.bufpos; int nextJump = gobbleMaxWithoutEscape(bpos, len - readChars); int newlyReadChars = nextJump - bpos; @@ -82,9 +72,9 @@ public class EscapeAwareReader extends Reader { if (newlyReadChars != 0) { if (cbuf != null) { - System.arraycopy(input, bpos, cbuf, off + readChars, newlyReadChars); + input.getChars(bpos, cbuf, off + readChars, newlyReadChars); } - } else if (nextJump == input.length) { + } else if (nextJump == input.length()) { // eof break; } @@ -100,7 +90,7 @@ public class EscapeAwareReader extends Reader { * the bufpos to where we should start the next jump. */ protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { - return this.bufpos = min(bufpos + maxReadahead, input.length); + return this.bufpos = min(bufpos + maxReadahead, input.length()); } protected void recordEscape(final int startOffsetInclusive, int length) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java index 8e39d172d3..ecd74755b3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.lang.ast.impl.io; +import net.sourceforge.pmd.util.document.Chars; + /** * Records where escapes occurred in the input document. This is quite * an inefficient way to deal with it, yet in the common case where there @@ -94,18 +96,18 @@ class EscapeTracker { * identified by the {@link #escapeRecords}. It must hold * that buf.length is the original source length. */ - private final char[] buf; + private final Chars buf; private int mark = Integer.MAX_VALUE; private int markEscape; private int markOutOffset; - Cursor(char[] buf) { + Cursor(Chars buf) { this.buf = buf; } char next() { - char c = buf[pos]; + char c = buf.charAt(pos); if (nextEscape < escapeRecords.length && pos == escapeRecords[nextEscape]) { pos += escapeRecords[nextEscape + 1]; // add escape length diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java index a3bbcd4dfa..d5b84c7c1d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java @@ -9,6 +9,8 @@ import static java.lang.Integer.min; import java.io.BufferedReader; import java.io.IOException; +import net.sourceforge.pmd.util.document.Chars; + /** * An implementation of java.io.Reader that translates Java unicode escapes. * This implementation has efficient block IO but poor char-by-char performance. @@ -25,15 +27,10 @@ public final class JavaInputReader extends EscapeAwareReader { */ private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - public JavaInputReader(CharSequence input, int startIdxInclusive, int endIdxExclusive) { - super(input, startIdxInclusive, endIdxExclusive); - } - - public JavaInputReader(CharSequence input) { + public JavaInputReader(Chars input) { super(input); } - /** * Returns the max offset, EXclusive, with which we can cut the input * array from the bufpos to dump it into the output array. This sets @@ -42,10 +39,10 @@ public final class JavaInputReader extends EscapeAwareReader { @Override protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { int off = bufpos; - int max = min(bufpos + maxReadahead, input.length); + int max = min(bufpos + maxReadahead, input.length()); boolean noBackSlash = false; int notEscapeEnd = this.savedNotEscapeSpecialEnd; - while (off < max && (noBackSlash = input[off] != '\\' || notEscapeEnd < off)) { + while (off < max && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { off++; } @@ -55,15 +52,15 @@ public final class JavaInputReader extends EscapeAwareReader { } final int firstBslashOff = off; - while (off < input.length && input[off] == '\\') { + while (off < input.length() && input.charAt(off) == '\\') { off++; } int bslashCount = off - firstBslashOff; // this condition is "is there an escape at offset firstBslashOff" if ((bslashCount & 1) == 1 // odd number of backslashes - && off < input.length - 4 // at least 5 chars to form the escape ('u' + 4 hex digits) - && input[off] == 'u') { // the char after the last backslash is a 'u' + && off < input.length() - 4 // at least 5 chars to form the escape ('u' + 4 hex digits) + && input.charAt(off) == 'u') { // the char after the last backslash is a 'u' replaceFirstBackslashWithEscape(firstBslashOff, off); this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; @@ -84,12 +81,12 @@ public final class JavaInputReader extends EscapeAwareReader { private void replaceFirstBackslashWithEscape(int posOfFirstBackSlash, int offOfTheU) throws IOException { try { char c = (char) - ( hexVal(input[++offOfTheU]) << 12 - | hexVal(input[++offOfTheU]) << 8 - | hexVal(input[++offOfTheU]) << 4 - | hexVal(input[++offOfTheU]) + ( hexVal(input.charAt(++offOfTheU)) << 12 + | hexVal(input.charAt(++offOfTheU)) << 8 + | hexVal(input.charAt(++offOfTheU)) << 4 + | hexVal(input.charAt(++offOfTheU)) ); - input[posOfFirstBackSlash] = c; // replace the start char of the backslash + input.set(posOfFirstBackSlash, c); // replace the start char of the backslash } catch (NumberFormatException e) { String message = "Invalid escape sequence at line " diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java index 18fa601b90..3b50b6433e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java @@ -15,15 +15,17 @@ public class NewCharStream implements CharStream { private final JavaccTokenDocument document; private final Cursor cursor; - public NewCharStream(JavaccTokenDocument document, EscapeTracker.Cursor cursor) { + private NewCharStream(JavaccTokenDocument document, EscapeTracker.Cursor cursor) { this.document = document; this.cursor = cursor; } - public static CharStream consume(EscapeAwareReader reader, JavaccTokenDocument doc) throws IOException { - try (EscapeAwareReader r = reader) { + public static CharStream open(JavaccTokenDocument doc) { + try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { reader.translate(); return new NewCharStream(doc, reader.escapes.new Cursor(reader.input)); + } catch (IOException e) { + throw new RuntimeException(e); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamFactory.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamFactory.java deleted file mode 100644 index dc1f0ce238..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamFactory.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; - -import java.io.IOException; -import java.io.Reader; -import java.util.function.Function; - -import org.apache.commons.io.IOUtils; - -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.CpdCompat; - -public final class CharStreamFactory { - - private CharStreamFactory() { - // util class - } - - /** - * A char stream that doesn't perform any escape translation. - */ - public static CharStream simpleCharStream(Reader input) { - return simpleCharStream(input, JavaccTokenDocument::new); - } - - /** - * A char stream that doesn't perform any escape translation. - */ - public static CharStream simpleCharStream(Reader input, Function documentMaker) { - String source = toString(input); - JavaccTokenDocument document = documentMaker.apply(TextDocument.readOnlyString(source, CpdCompat.dummyVersion())); - return new SimpleCharStream(document); - } - - /** - * A char stream that translates java unicode sequences. - */ - public static CharStream javaCharStream(Reader input) { - return javaCharStream(input, JavaccTokenDocument::new); - } - - /** - * A char stream that translates java unicode sequences. - */ - public static CharStream javaCharStream(Reader input, Function documentMaker) { - String source = toString(input); - JavaccTokenDocument tokens = documentMaker.apply(TextDocument.readOnlyString(source, CpdCompat.dummyVersion())); - return new JavaCharStream(tokens); - } - - /** - * @deprecated This shouldn't be used. IOExceptions should be handled properly, - * ie it should be expected that creating a parse may throw an IOException, - * in both CPD and PMD - */ - @Deprecated - public static String toString(Reader dstream) { - try (Reader r = dstream) { - return IOUtils.toString(r); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaCharStream.java deleted file mode 100644 index 87f7d6b059..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaCharStream.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; - -import java.io.EOFException; -import java.io.IOException; - -import net.sourceforge.pmd.util.document.Chars; - -/** - * This stream buffers the whole file in memory before parsing, - * and track start/end offsets of tokens. This allows building {@link JavaccToken}. - * The buffer is assumed to be composed of only ASCII characters, - * and the stream unescapes Unicode escapes. The {@link #getTokenDocument() token document} - * stores the original file with escapes and all. - */ -public class JavaCharStream extends JavaCharStreamBase { - - // full text with nothing escaped and all - private final Chars fullText; - private final JavaccTokenDocument document; - - private int[] startOffsets; - - public JavaCharStream(JavaccTokenDocument document) { - super(document.getTextDocument().newReader()); - this.fullText = document.getFullText(); - this.document = document; - this.startOffsets = new int[bufsize]; - maxNextCharInd = fullText.length(); - - nextCharBuf = null; - } - - @Override - protected void ExpandBuff(boolean wrapAround) { - int[] newStartOffsets = new int[bufsize + 2048]; - - if (wrapAround) { - System.arraycopy(startOffsets, tokenBegin, newStartOffsets, 0, bufsize - tokenBegin); - System.arraycopy(startOffsets, 0, newStartOffsets, bufsize - tokenBegin, bufpos); - startOffsets = newStartOffsets; - } else { - System.arraycopy(startOffsets, tokenBegin, newStartOffsets, 0, bufsize - tokenBegin); - startOffsets = newStartOffsets; - } - - super.ExpandBuff(wrapAround); - } - - @Override - protected void UpdateLineColumn(char c) { - startOffsets[bufpos] = nextCharInd; - super.UpdateLineColumn(c); - } - - @Override - public int getStartOffset() { - return startOffsets[tokenBegin]; - } - - @Override - public int getEndOffset() { - if (isAtEof()) { - return fullText.length(); - } else { - return startOffsets[bufpos] + 1; // + 1 for exclusive - } - } - - @Override - public JavaccTokenDocument getTokenDocument() { - return document; - } - - @Override - public String GetImage() { - if (bufpos >= tokenBegin) { - return new String(buffer, tokenBegin, bufpos - tokenBegin + 1); - } else { - return new String(buffer, tokenBegin, bufsize - tokenBegin) - + new String(buffer, 0, bufpos + 1); - } - } - - @Override - protected char ReadByte() throws IOException { - ++nextCharInd; - - if (isAtEof()) { - if (bufpos != 0) { - --bufpos; - if (bufpos < 0) { - bufpos += bufsize; - } - } else { - bufline[bufpos] = line; - bufcolumn[bufpos] = column; - startOffsets[bufpos] = fullText.length(); - } - throw new EOFException(); - } - - return fullText.charAt(nextCharInd); - } - - private boolean isAtEof() { - return nextCharInd >= fullText.length(); - } - - - @Override - protected void FillBuff() { - throw new IllegalStateException("Buffer shouldn't be refilled"); - } - -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index bb5c3c608c..8990aacd04 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -9,6 +9,9 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; +import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -23,6 +26,16 @@ public class JavaccTokenDocument extends TokenDocument { super(textDocument); } + + public EscapeAwareReader newReader(Chars text) { + return new EscapeAwareReader(text); + } + + public CharStream newCharStream() { + return NewCharStream.open(this); + } + + /** * Open the document. This is only meant to be used by a Javacc-generated * parser. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/SimpleCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/SimpleCharStream.java deleted file mode 100644 index d453b34797..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/SimpleCharStream.java +++ /dev/null @@ -1,20 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; - -/** - * A char stream that does not perform unicode escaping. - */ -public class SimpleCharStream extends JavaCharStream { - - public SimpleCharStream(JavaccTokenDocument document) { - super(document); - } - - @Override - protected boolean doEscape() { - return false; - } -} diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 5b145f05a7..6aa98771b6 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -6,18 +6,21 @@ package net.sourceforge.pmd.cpd; import java.io.BufferedReader; import java.io.IOException; -import java.io.Reader; -import java.io.StringReader; import java.util.Properties; +import org.checkerframework.checker.nullness.qual.Nullable; + import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; -import net.sourceforge.pmd.lang.cpp.ast.CppCharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; -import net.sourceforge.pmd.util.IOUtil; +import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; /** * The C++ tokenizer. @@ -32,6 +35,12 @@ public class CPPTokenizer extends JavaCCTokenizer { setProperties(new Properties()); // set the defaults } + // override to make it visible in tests + @Override + protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { + return super.getLexerForSource(sourceCode); + } + /** * Sets the possible options for the C++ tokenizer. * @@ -54,12 +63,12 @@ public class CPPTokenizer extends JavaCCTokenizer { } } - private String maybeSkipBlocks(String test) throws IOException { + private Chars maybeSkipBlocks(Chars test) throws IOException { if (!skipBlocks) { return test; } - try (BufferedReader reader = new BufferedReader(new StringReader(test))) { + try (BufferedReader reader = new BufferedReader(test.newReader())) { StringBuilder filtered = new StringBuilder(test.length()); String line; boolean skip = false; @@ -75,26 +84,28 @@ public class CPPTokenizer extends JavaCCTokenizer { // always add a new line to keep the line-numbering filtered.append(PMD.EOL); } - return filtered.toString(); + return Chars.wrap(filtered, false); } } @Override - protected CharStream makeCharStream(Reader sourceCode) { - return CppCharStream.newCppCharStream(sourceCode); + protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + return new JavaccTokenDocument(textDoc) { + @Override + public EscapeAwareReader newReader(Chars text) { + return new CppEscapeReader(text); + } + + @Override + protected @Nullable String describeKindImpl(int kind) { + return CppTokenKinds.describe(kind); + } + }; } @Override protected TokenManager makeLexerImpl(CharStream sourceCode) { return CppTokenKinds.newTokenManager(sourceCode); } - - @SuppressWarnings("PMD.CloseResource") - @Override - protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { - Reader reader = IOUtil.skipBOM(new StringReader(maybeSkipBlocks(sourceCode.getCodeBuffer().toString()))); - CharStream charStream = makeCharStream(reader); - return makeLexerImpl(charStream); - } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java deleted file mode 100644 index c7f7aec7c2..0000000000 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStream.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.cpp.ast; - -import java.io.IOException; -import java.io.Reader; -import java.util.regex.Pattern; - -import org.checkerframework.checker.nullness.qual.Nullable; - -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.SimpleCharStream; -import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.CpdCompat; - -/** - * A SimpleCharStream, that supports the continuation of lines via backslash+newline, - * which is used in C/C++. - * - * @author Andreas Dangel - */ -public class CppCharStream extends SimpleCharStream { - - private static final Pattern CONTINUATION = Pattern.compile("\\\\\\n|\\\\\\r\\n"); - private static final char BACKSLASH = '\\'; - private static final char NEWLINE = '\n'; - private static final char CARRIAGE_RETURN = '\r'; - - CppCharStream(JavaccTokenDocument document) { - super(document); - } - - - @Override - public char readChar() throws IOException { - char c = super.readChar(); - if (c == BACKSLASH) { - char c1 = super.readChar(); - if (c1 == NEWLINE) { - c = super.readChar(); - } else if (c1 == CARRIAGE_RETURN) { - char c2 = super.readChar(); - if (c2 == NEWLINE) { - c = super.readChar(); - } else { - backup(2); - } - } else { - backup(1); - } - } - return c; - } - - @Override - public char[] GetSuffix(int len) { - String image = GetImage(); - return image.substring(image.length() - len, image.length()).toCharArray(); - } - - @Override - public String GetImage() { - String image = super.GetImage(); - return CONTINUATION.matcher(image).replaceAll(""); - } - - public static CppCharStream newCppCharStream(Reader dstream) { - String source = CharStreamFactory.toString(dstream); - JavaccTokenDocument document = new JavaccTokenDocument(TextDocument.readOnlyString(source, CpdCompat.dummyVersion())) { - @Override - protected @Nullable String describeKindImpl(int kind) { - return CppTokenKinds.describe(kind); - } - }; - return new CppCharStream(document); - } -} diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java new file mode 100644 index 0000000000..378dfb1011 --- /dev/null +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -0,0 +1,61 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.cpp.ast; + +import static java.lang.Integer.min; + +import java.io.IOException; + +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.util.document.Chars; + +public class CppEscapeReader extends EscapeAwareReader { + + private static final char NEWLINE = '\n'; + private static final char CARRIAGE_RETURN = '\r'; + + private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; + + public CppEscapeReader(Chars input) { + super(input); + } + + @Override + protected int gobbleMaxWithoutEscape(int bufpos, int maxReadahead) throws IOException { + int off = bufpos; + int max = min(bufpos + maxReadahead, input.length()); + boolean noBackSlash = false; + int notEscapeEnd = this.savedNotEscapeSpecialEnd; + while (off < max && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { + off++; + } + + if (noBackSlash) { + this.bufpos = off; + return off; + } + + final int backSlackOff = off++; + if (input.charAt(off) == NEWLINE) { + recordEscape(backSlackOff, 2); + this.bufpos = off + 2; + return backSlackOff; + } else if (input.charAt(off) == CARRIAGE_RETURN) { + if (input.charAt(++off) == NEWLINE) { + recordEscape(backSlackOff, 3); + this.bufpos = off + 3; + return backSlackOff; + } + } + + // not an escape sequence + int min = min(bufpos + maxReadahead, off); + // save the number of backslashes that are part of the escape, + // might have been cut in half by the maxReadahead + this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; + this.bufpos = min; + return min; + } +} diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java similarity index 61% rename from pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStreamTest.java rename to pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 8196b4b079..9d6df7dd49 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/lang/cpp/ast/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -2,38 +2,47 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.cpp.ast; +package net.sourceforge.pmd.cpd; import static org.junit.Assert.assertEquals; import java.io.IOException; -import java.io.StringReader; +import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; +import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; +import net.sourceforge.pmd.util.document.TextDocument; + public class CppCharStreamTest { + @NonNull + public CharStream charStreamFor(String source) { + return NewCharStream.open(new CPPTokenizer().newTokenDoc(TextDocument.readOnlyString(source))); + } + @Test public void testContinuationUnix() throws IOException { - CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\\nb")); + CharStream stream = charStreamFor("a\\\nb"); assertStream(stream, "ab"); } @Test public void testContinuationWindows() throws IOException { // note that the \r is normalized to a \n by the TextFile - CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\\r\nb")); + CharStream stream = charStreamFor("a\\\r\nb"); assertStream(stream, "ab"); } @Test public void testBackup() throws IOException { // note that the \r is normalized to a \n by the TextFile - CppCharStream stream = CppCharStream.newCppCharStream(new StringReader("a\\b\\qc")); + CharStream stream = charStreamFor("a\\b\\\rc"); assertStream(stream, "a\\b\\qc"); } - private void assertStream(CppCharStream stream, String token) throws IOException { + private void assertStream(CharStream stream, String token) throws IOException { char c = stream.BeginToken(); assertEquals(token.charAt(0), c); for (int i = 1; i < token.length(); i++) { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index b57c509d2e..83c69ae100 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -5,7 +5,6 @@ package net.sourceforge.pmd.cpd; import java.io.IOException; -import java.io.Reader; import java.util.Deque; import java.util.LinkedList; import java.util.Properties; @@ -15,10 +14,11 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds; +import net.sourceforge.pmd.util.document.TextDocument; public class JavaTokenizer extends JavaCCTokenizer { @@ -44,8 +44,8 @@ public class JavaTokenizer extends JavaCCTokenizer { } @Override - protected CharStream makeCharStream(Reader sourceCode) { - return CharStreamFactory.javaCharStream(sourceCode, InternalApiBridge::javaTokenDoc); + protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + return InternalApiBridge.javaTokenDoc(textDoc); } @Override diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index d7af60de77..a3bf759260 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.lang.java.ast; import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaCharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker; @@ -32,11 +31,6 @@ public class JavaParser extends JjtreeParserAdapter { return new JavaTokenDocument(textDocument); } - @Override - protected CharStream newCharStream(JavaccTokenDocument tokenDocument) { - return new JavaCharStream(tokenDocument); - } - @Override protected ASTCompilationUnit parseImpl(CharStream cs, ParserTask task) throws ParseException { JavaParserImpl parser = new JavaParserImpl(cs); diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index e00a9102d6..a3a67d2fb0 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -15,8 +15,11 @@ import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.WHITESPACE; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -43,6 +46,11 @@ final class JavaTokenDocument extends JavaccTokenDocument { } + @Override + public EscapeAwareReader newReader(Chars text) { + return new JavaInputReader(text); + } + @Override protected @Nullable String describeKindImpl(int kind) { return JavaTokenKinds.describe(kind); diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index b6383affca..fcb65f40f2 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -4,15 +4,16 @@ package net.sourceforge.pmd.cpd; -import java.io.IOException; -import java.io.Reader; - import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; +import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; public class JSPTokenizer extends JavaCCTokenizer { @@ -22,7 +23,13 @@ public class JSPTokenizer extends JavaCCTokenizer { } @Override - protected CharStream makeCharStream(Reader sourceCode) throws IOException { - return CharStreamFactory.javaCharStream(sourceCode); + protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + return new JavaccTokenDocument(textDoc) { + @Override + public EscapeAwareReader newReader(Chars text) { + return new JavaInputReader(text); + } + }; } + } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java index 65cb990593..ed2cb3e08d 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java @@ -11,7 +11,6 @@ import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.TextDocument; public class PLSQLParser extends JjtreeParserAdapter { diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 8e9a9a3d46..9e61ea2067 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -4,7 +4,6 @@ package net.sourceforge.pmd.cpd; -import java.io.Reader; import java.util.regex.Pattern; import org.checkerframework.checker.nullness.qual.Nullable; @@ -12,7 +11,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds; @@ -31,8 +29,8 @@ public class PythonTokenizer extends JavaCCTokenizer { } @Override - protected CharStream makeCharStream(Reader sourceCode) { - return CharStreamFactory.simpleCharStream(sourceCode, PythonTokenDocument::new); + protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + return new PythonTokenDocument(textDoc); } private static class PythonTokenDocument extends JavaccTokenDocument { diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index f4166db699..8fa9212144 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -4,15 +4,16 @@ package net.sourceforge.pmd.cpd; -import java.io.IOException; -import java.io.Reader; - import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStreamFactory; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; +import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; /** * @author sergey.gorbaty @@ -25,7 +26,13 @@ public class VfTokenizer extends JavaCCTokenizer { } @Override - protected CharStream makeCharStream(Reader sourceCode) throws IOException { - return CharStreamFactory.javaCharStream(sourceCode); + protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + return new JavaccTokenDocument(textDoc) { + @Override + public EscapeAwareReader newReader(Chars text) { + return new JavaInputReader(text); + } + }; } + } From d3a434a933182602e187152295aedad751577d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 07:31:46 +0200 Subject: [PATCH 004/136] Remove pmd core ant sources --- pmd-core/pom.xml | 23 ------ pmd-core/src/main/ant/alljavacc.xml | 122 ---------------------------- 2 files changed, 145 deletions(-) delete mode 100644 pmd-core/src/main/ant/alljavacc.xml diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index 1cdde6d0c6..4ac0e97326 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -13,29 +13,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - true - - - generate-sources - generate-sources - - - - - - - - - - run - - - - - org.codehaus.mojo build-helper-maven-plugin diff --git a/pmd-core/src/main/ant/alljavacc.xml b/pmd-core/src/main/ant/alljavacc.xml deleted file mode 100644 index c5e34e728b..0000000000 --- a/pmd-core/src/main/ant/alljavacc.xml +++ /dev/null @@ -1,122 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Using JavaCC home: ${javacc-home.path} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 2e725038ac825bc69feefb2875dc7f449c847982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 07:58:02 +0200 Subject: [PATCH 005/136] Checkout tests --- .../lang/ast/impl/io/EscapeAwareReader.java | 5 +- .../pmd/lang/ast/impl/io/EscapeTracker.java | 26 +++- .../pmd/lang/ast/impl/io/JavaInputReader.java | 5 - .../pmd/lang/ast/impl/io/NewCharStream.java | 26 ++-- .../ast/impl/javacc/JavaccTokenDocument.java | 5 - .../ast/impl/javacc/JjtreeParserAdapter.java | 11 +- .../lang/ast/impl/io/JavaInputReaderTest.java | 143 ++++++++++++++++++ .../ast/impl/javacc/CharStreamImplTest.java | 143 ++++++++++++++++++ 8 files changed, 332 insertions(+), 32 deletions(-) create mode 100644 pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java create mode 100644 pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java index 69ee25d9d7..51e97a70f1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java @@ -8,7 +8,6 @@ import static java.lang.Integer.min; import java.io.IOException; import java.io.Reader; -import java.nio.CharBuffer; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; @@ -135,11 +134,11 @@ public class EscapeAwareReader extends Reader { } public int getLine(int idxInInput) { - return StringUtil.lineNumberAt(CharBuffer.wrap(input), idxInInput); + return StringUtil.lineNumberAt(input, idxInInput); } public int getColumn(int idxInInput) { - return StringUtil.columnNumberAt(CharBuffer.wrap(input), idxInInput); + return StringUtil.columnNumberAt(input, idxInInput); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java index ecd74755b3..8ed93855da 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.lang.ast.impl.io; +import java.io.EOFException; + import net.sourceforge.pmd.util.document.Chars; /** @@ -106,7 +108,11 @@ class EscapeTracker { this.buf = buf; } - char next() { + char next() throws EOFException { + if (pos == buf.length()) { + throw new EOFException(); + } + char c = buf.charAt(pos); if (nextEscape < escapeRecords.length && pos == escapeRecords[nextEscape]) { @@ -120,6 +126,12 @@ class EscapeTracker { } void backup(int numChars) { + ensureMarked(); + if (numChars > markLength()) { + throw new IllegalArgumentException( + "Cannot backup " + numChars + " chars, only " + markLength() + " are saved"); + } + outOffset -= numChars; if (nextEscape <= 0) { @@ -149,8 +161,7 @@ class EscapeTracker { } void markToString(StringBuilder sb) { - assert mark <= pos : "Wrong mark"; - assert markEscape <= nextEscape : "Wrong mark"; + ensureMarked(); int prevLength = sb.length(); @@ -175,6 +186,15 @@ class EscapeTracker { } } + private void ensureMarked() { + if (mark == Integer.MAX_VALUE) { + throw new IllegalStateException("Mark is not set"); + } + assert mark <= pos : "Wrong mark"; + assert markEscape <= nextEscape : "Wrong mark"; + assert markEscape <= escapeRecords.length : "Wrong escape mark"; + } + int curOutOffset() { return outOffset; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java index d5b84c7c1d..b41211c0ed 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java @@ -31,11 +31,6 @@ public final class JavaInputReader extends EscapeAwareReader { super(input); } - /** - * Returns the max offset, EXclusive, with which we can cut the input - * array from the bufpos to dump it into the output array. This sets - * the bufpos to where we should start the next jump. - */ @Override protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { int off = bufpos; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java index 3b50b6433e..dd8ded5daf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java @@ -4,28 +4,28 @@ package net.sourceforge.pmd.lang.ast.impl.io; +import java.io.EOFException; import java.io.IOException; import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeTracker.Cursor; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.util.document.FileLocation; +import net.sourceforge.pmd.util.document.TextDocument; public class NewCharStream implements CharStream { private final JavaccTokenDocument document; - private final Cursor cursor; + private final EscapeTracker.Cursor cursor; private NewCharStream(JavaccTokenDocument document, EscapeTracker.Cursor cursor) { this.document = document; this.cursor = cursor; } - public static CharStream open(JavaccTokenDocument doc) { + public static CharStream open(JavaccTokenDocument doc) throws IOException { try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { reader.translate(); return new NewCharStream(doc, reader.escapes.new Cursor(reader.input)); - } catch (IOException e) { - throw new RuntimeException(e); } } @@ -35,15 +35,14 @@ public class NewCharStream implements CharStream { } @Override - public char readChar() { + public char readChar() throws EOFException { return cursor.next(); } @Override - public char BeginToken() { - char c = cursor.next(); + public char BeginToken() throws EOFException { cursor.mark(); - return c; + return cursor.next(); } @Override @@ -60,12 +59,17 @@ public class NewCharStream implements CharStream { @Override public int getEndColumn() { - return 0; // TODO + return endLocation().getEndColumn(); } @Override public int getEndLine() { - return 0; // TODO + return endLocation().getEndLine(); + } + + private FileLocation endLocation() { + TextDocument textDoc = document.getTextDocument(); + return textDoc.toLocation(textDoc.createRegion(getEndOffset(), 0)); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 8990aacd04..8c0e82ce4a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -10,7 +10,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -31,10 +30,6 @@ public class JavaccTokenDocument extends TokenDocument { return new EscapeAwareReader(text); } - public CharStream newCharStream() { - return NewCharStream.open(this); - } - /** * Open the document. This is only meant to be used by a Javacc-generated diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index 3d8b2c9afc..9efb6dc342 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -4,11 +4,14 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; +import java.io.IOException; + import net.sourceforge.pmd.lang.Parser; import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.RootNode; import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -26,17 +29,15 @@ public abstract class JjtreeParserAdapter implements Parser protected abstract JavaccTokenDocument newDocumentImpl(TextDocument textDocument); - protected CharStream newCharStream(JavaccTokenDocument tokenDocument) { - return new SimpleCharStream(tokenDocument); - } - @Override public R parse(ParserTask task) throws ParseException { JavaccTokenDocument doc = newDocumentImpl(task.getTextDocument()); - CharStream charStream = newCharStream(doc); try { + CharStream charStream = NewCharStream.open(doc); return parseImpl(charStream, task); + } catch (IOException e) { + throw new TokenMgrError(-1, -1, fileName, "IO error", e); } catch (TokenMgrError tme) { throw tme.setFileName(task.getFileDisplayName()); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java new file mode 100644 index 0000000000..0f03a7bed9 --- /dev/null +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java @@ -0,0 +1,143 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.io; + +import java.io.IOException; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.junit.Assert; +import org.junit.Test; + +import net.sourceforge.pmd.util.document.Chars; + + +public class JavaInputReaderTest { + + @NonNull + public JavaInputReader readString(String input) { + return new JavaInputReader(Chars.wrap(input, true)); + } + + + @Test + public void testSimpleRead() throws IOException { + + String input = "abcdede"; + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[12]; + + int read = r.read(chars, 0, chars.length); + + Assert.assertEquals("Should have read the entire text", input.length(), read); + Assert.assertEquals(input, new String(chars, 0, input.length())); + } + } + + @Test + public void testNotAnEscape1Read() throws IOException { + + String input = "abc\\dede"; + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[12]; + + int read = r.read(chars, 0, chars.length); + + Assert.assertEquals("Should have read the entire text", input.length(), read); + Assert.assertEquals(input, new String(chars, 0, input.length())); + } + } + + @Test + public void testNotAnEscape1Read2() throws IOException { + + String input = "abc\\\\\\dede"; + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[12]; + + int read = r.read(chars, 0, chars.length); + + Assert.assertEquals("Should have read the entire text", input.length(), read); + Assert.assertEquals(input, new String(chars, 0, input.length())); + } + } + + @Test + public void testNotAnEscape1Read3_SplitInTheMiddleOfBackslashes() throws IOException { + + String input = "abc\\\\\\dede"; + // ^ + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[12]; + + int read = r.read(chars, 0, 4); + + Assert.assertEquals("Should have read just enough", 4, read); + assertBufferIsJust("abc\\", chars, 0); + + read = r.read(chars, 4, 1); + + Assert.assertEquals(1, read); + assertBufferIsJust("abc\\\\", chars, 0); + + read = r.read(chars, 5, 8); + + Assert.assertEquals(5, read); + assertBufferIsJust("abc\\\\\\dede", chars, 0); + } + } + + @Test + public void testAnEscapeStopAtEnd() throws IOException { + + String input = "abc\\\\\\u00a0dede"; + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[12]; + + int read = r.read(chars, 0, 4); + + Assert.assertEquals(4, read); + assertBufferIsJust("abc\u00a0", chars, 0); + + read = r.read(chars, 4, 2); + + Assert.assertEquals(2, read); + assertBufferIsJust("abc\u00a0de", chars, 0); + } + } + + @Test + public void testAnEscapeInsideBlock() throws IOException { + + String input = "abc\\\\\\u00a0dede\\u00a0"; + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[12]; + + int read = r.read(chars, 0, 12); + + Assert.assertEquals(9, read); + assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); + + read = r.read(chars, 9, 10); + + Assert.assertEquals(-1, read); + assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); + } + } + + private static void assertBufferIsJust(String contents, char[] chars, int off) { + // asserts the rest of the buffer is null characters + char[] chars2 = new char[chars.length]; + contents.getChars(0, contents.length(), chars2, off); + Assert.assertArrayEquals(chars2, chars); + } + + +} \ No newline at end of file diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java new file mode 100644 index 0000000000..5eec3ba4d9 --- /dev/null +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java @@ -0,0 +1,143 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +import java.io.EOFException; +import java.io.IOException; + +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.ExpectedException; + +import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; +import net.sourceforge.pmd.util.document.TextDocument; + +public class CharStreamImplTest { + + @Rule + public ExpectedException expect = ExpectedException.none(); + + @Test + public void testReadZeroChars() throws IOException { + + CharStream stream = getCharStream(""); + + expect.expect(EOFException.class); + + try { + stream.readChar(); + } catch (Exception e) { + assertEquals(stream.getStartOffset(), 0); + assertEquals(stream.getEndOffset(), 0); + throw e; + } + } + + @Test + public void testReadEofChars() throws IOException { + + CharStream stream = getCharStream(""); + + expect.expect(EOFException.class); + + try { + stream.readChar(); + } catch (Exception e) { + assertEquals(stream.getStartOffset(), 0); + assertEquals(stream.getEndOffset(), 0); + throw e; + } + } + + @Test + public void testMultipleEofReads() throws IOException { + + CharStream stream = getCharStream(""); + + for (int i = 0; i < 3; i++) { + try { + stream.readChar(); + fail(); + } catch (EOFException ignored) { + + } + } + + } + + @Test + public void testReadStuff() throws IOException { + + CharStream stream = getCharStream("abcd"); + + assertEquals('a', stream.readChar()); + assertEquals('b', stream.readChar()); + assertEquals('c', stream.readChar()); + assertEquals('d', stream.readChar()); + + expect.expect(EOFException.class); + stream.readChar(); + } + + @Test + public void testReadBacktrack() throws IOException { + + CharStream stream = getCharStream("abcd"); + + assertEquals('a', stream.BeginToken()); + assertEquals('b', stream.readChar()); + assertEquals('c', stream.readChar()); + assertEquals('d', stream.readChar()); + + assertEquals("abcd", stream.GetImage()); + + stream.backup(2); + assertEquals('c', stream.readChar()); + assertEquals('d', stream.readChar()); + + + expect.expect(EOFException.class); + stream.readChar(); + } + + public CharStream getCharStream(String abcd) throws IOException { + return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd))); + } + + @Test + public void testBacktrackTooMuch() throws IOException { + + CharStream stream = getCharStream("abcd"); + + assertEquals('a', stream.readChar()); + assertEquals('b', stream.readChar()); + assertEquals('c', stream.BeginToken()); + assertEquals('d', stream.readChar()); + + expect.expect(IllegalArgumentException.class); + expect.expectMessage("only 2 are saved"); + stream.backup(10); + } + @Test + public void testBacktrackTooMuch2() throws IOException { + + CharStream stream = getCharStream("abcd"); + + assertEquals('a', stream.BeginToken()); + assertEquals('b', stream.readChar()); + assertEquals('c', stream.readChar()); + assertEquals('d', stream.readChar()); + + expect.expect(IllegalArgumentException.class); + expect.expectMessage("only 4 are saved"); + stream.backup(10); + } + + +} From d4be567652fd5f73a5bcab8f18c295e22db1f9af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 09:52:28 +0200 Subject: [PATCH 006/136] Fix zero-length escapes --- .../lang/ast/impl/io/EscapeAwareReader.java | 37 +++--- .../pmd/lang/ast/impl/io/EscapeTracker.java | 105 ++++++++++++------ .../pmd/lang/ast/impl/io/JavaInputReader.java | 13 +-- .../lang/ast/impl/io/JavaInputReaderTest.java | 4 +- .../pmd/lang/cpp/ast/CppEscapeReader.java | 19 ++-- .../pmd/cpd/CppCharStreamTest.java | 2 +- 6 files changed, 109 insertions(+), 71 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java index 51e97a70f1..fa8412e920 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java @@ -13,11 +13,11 @@ import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; /** - * A reader that optionally escapes its input text. It records where + * A reader that can interpret escapes in its input text. It records where * escapes occurred, and can translate an offset in the translated * input document to a line+column position in the original input. * - *

The default implementation does not perform any escaping. + *

The default implementation does not perform any escape translation. */ @SuppressWarnings("PMD.AssignmentInOperand") public class EscapeAwareReader extends Reader { @@ -27,12 +27,6 @@ public class EscapeAwareReader extends Reader { * first backslash is replaced with the translated value of the * escape. The bufpos is updated so that we read the next char * after the escape. - * - *

This makes it so that 1. we don't need an additional buffer for - * translated chars, and 2. the full escape is preserved, just use - * the {@link EscapeTracker} to get the position of the escape and - * replace the first char with a backslash. We can report unnecessary - * escapes that way. */ protected Chars input; /** Position of the next char to read in the input. */ @@ -50,21 +44,30 @@ public class EscapeAwareReader extends Reader { * Translate all the characters in the buffer. */ public int translate() throws IOException { - return read(null, 0, Integer.MAX_VALUE); + return readUnchecked(null, 0, Integer.MAX_VALUE); } @Override - public int read(final char[] cbuf, final int off, final int len) throws IOException { + public int read(final char[] cbuf, final int off, int len) throws IOException { + if (off < 0 || len < 0 || len + off > cbuf.length) { + throw new IndexOutOfBoundsException("cbuf len=" + cbuf.length + " off=" + off + " len=" + len); + } + return readUnchecked(cbuf, off, len); + } + + private int readUnchecked(char[] cbuf, int off, int len) throws IOException { ensureOpen(); if (this.bufpos == input.length()) { return -1; } + len = min(len, input.length()); // remove Integer.MAX_VALUE + int readChars = 0; while (readChars < len && this.bufpos < input.length()) { int bpos = this.bufpos; - int nextJump = gobbleMaxWithoutEscape(bpos, len - readChars); + int nextJump = gobbleMaxWithoutEscape(min(input.length(), bpos + len - readChars)); int newlyReadChars = nextJump - bpos; assert newlyReadChars >= 0 && (readChars + newlyReadChars) <= len; @@ -82,18 +85,20 @@ public class EscapeAwareReader extends Reader { return readChars; } - /** * Returns the max offset, EXclusive, with which we can cut the input * array from the bufpos to dump it into the output array. This sets * the bufpos to where we should start the next jump. */ - protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { - return this.bufpos = min(bufpos + maxReadahead, input.length()); + protected int gobbleMaxWithoutEscape(int maxOff) throws IOException { + return this.bufpos = maxOff; } - protected void recordEscape(final int startOffsetInclusive, int length) { - this.escapes.recordEscape(startOffsetInclusive, length); + protected int recordEscape(final int startOffsetInclusive, int lengthInSource, int translatedLength) { + assert lengthInSource > 0 && startOffsetInclusive >= 0; + this.escapes.recordEscape(startOffsetInclusive, lengthInSource, translatedLength); + this.bufpos = startOffsetInclusive + lengthInSource; + return startOffsetInclusive + translatedLength; } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java index 8ed93855da..7b3a23798c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java @@ -16,43 +16,77 @@ import net.sourceforge.pmd.util.document.Chars; class EscapeTracker { private static final int[] EMPTY = new int[0]; + private static final int RECORD_SIZE = 3; - /** + /* * Offsets in the input buffer where a unicode escape occurred. - * Represented as pairs [off, len] where + * Represented as tuples (off, len, invalid) where * - off is the offset in the source file where the escape occurred - * - len is the length in characters of the escape (which is translated to a single char). + * - len is the length of the escape in the input file, eg for \ u 00a0 will be 6 + * - invalid is the last offset in the buffer which contains the translated chars (exclusive) + * + * Eg for "a\u00a0b" (translates as "a b"), the buffer looks like + * [a u00a0b] + * ^ this char has been replaced with the translated value of the escape + * ^^^^^ these characters are only present in the input, we jump over them when reading + * ^ off + * ^ invalid + * ^ off + len + * + * The escape record is (1,6,2) + * + * When reading the buffer we'll copy two blocks + * * "a " + * * then jump over "u00a0" and copy "b" + * + * In general to read until an escape means reading until its 'invalid' + * field, and once that is reached, jump to off + len. + * */ private int[] escapeRecords = EMPTY; /** Index of the next write in the {@link #escapeRecords}. */ private int nextFreeIdx = 0; + /** * Calls to this method must occur in source order (ie param * offsetInInput increases monotonically). */ - void recordEscape(int offsetInInput, int len) { + void recordEscape(int offsetInInput, int lengthInInput, int lengthInOutput) { if (nextFreeIdx + 1 >= escapeRecords.length) { - // double capacity, add 1 to not stay stuck at zero - int[] newOffsets = new int[(escapeRecords.length + 1) * 2]; + // add 1 to not stay stuck at zero + int[] newOffsets = new int[(escapeRecords.length + 1) * RECORD_SIZE]; System.arraycopy(escapeRecords, 0, newOffsets, 0, escapeRecords.length); this.escapeRecords = newOffsets; } escapeRecords[nextFreeIdx++] = offsetInInput; - escapeRecords[nextFreeIdx++] = len - 1; // -1 because the translated escape has length 1 + escapeRecords[nextFreeIdx++] = lengthInInput; + escapeRecords[nextFreeIdx++] = offsetInInput + lengthInOutput; + } + + private int inOff(int idx) { + return escapeRecords[idx]; + } + + private int inLen(int idx) { + return escapeRecords[idx + 1]; + } + + private int invalidIdx(int idx) { + return escapeRecords[idx + 2]; } /** * Convert an offset in the translated file into an offset in * the untranslated input. */ - public int inputOffsetAt(int translatedOffset) { + int inputOffsetAt(int translatedOffset) { // basically accumulate the lengths of all escapes occurring before the given translatedOffset int sum = translatedOffset; - for (int i = 0; i < nextFreeIdx; i += 2) { - if (escapeRecords[i] < sum) { - sum += escapeRecords[i + 1]; + for (int i = 0; i < maxEscape(); i += RECORD_SIZE) { + if (inOff(i) < sum) { + sum += inLen(i); } else { break; } @@ -60,16 +94,24 @@ class EscapeTracker { return sum; } + int maxEscape() { + return nextFreeIdx; + } + @Override public String toString() { StringBuilder res = new StringBuilder("Escape set {"); - for (int i = 0; i < nextFreeIdx; i += 2) { - res.append("(at=").append(escapeRecords[i]).append(", len=").append(escapeRecords[i + 1]).append("), "); + for (int i = 0; i < maxEscape(); i += RECORD_SIZE) { + res.append("(at=").append(inOff(i)) + .append(", inlen=").append(inLen(i)) + .append(", invalidAt=").append(invalidIdx(i)) + .append("), "); } return res.append('}').toString(); } + /** Backend for a CharStream. */ class Cursor { @@ -112,13 +154,14 @@ class EscapeTracker { if (pos == buf.length()) { throw new EOFException(); } + char c; - char c = buf.charAt(pos); - - if (nextEscape < escapeRecords.length && pos == escapeRecords[nextEscape]) { - pos += escapeRecords[nextEscape + 1]; // add escape length - this.nextEscape += 2; + if (nextEscape < maxEscape() && pos == invalidIdx(nextEscape)) { + pos += inLen(nextEscape); // add escape length + c = buf.charAt(pos); + this.nextEscape += RECORD_SIZE; } else { + c = buf.charAt(pos); pos++; } outOffset++; @@ -137,20 +180,20 @@ class EscapeTracker { if (nextEscape <= 0) { pos -= numChars; // then there were no escapes before the 'pos' } else { - int off = pos; - for (int i = nextEscape - 2; i >= 0 && numChars > 0; i -= 2) { - int esc = escapeRecords[i]; - if (esc == off) { - off -= escapeRecords[i + 1]; - } else if (esc > off) { + int inoff = pos; + for (int i = nextEscape - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { + int esc = inOff(i); + if (esc == inoff) { + inoff -= inLen(i); + } else if (esc > inoff) { // then the current escape was before what we're looking at break; } else { - off--; + inoff--; } numChars--; } - pos = off - numChars; + pos = inoff - numChars; } } @@ -174,15 +217,13 @@ class EscapeTracker { int cur = mark; int esc = markEscape; while (cur < pos && esc < nextEscape) { - int escapeOff = escapeRecords[esc]; - assert escapeOff < pos; - sb.append(buf, cur, escapeOff + 1); - cur = escapeOff + escapeRecords[esc + 1]; - esc += 2; + sb.append(buf, cur, invalidIdx(esc)); + cur = inOff(esc) + inLen(esc); + esc += RECORD_SIZE; } // no more escape in the range, append everything until the pos sb.append(buf, cur, pos + 1); - assert sb.length() - prevLength == markLength(); + assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength(); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java index b41211c0ed..53da0c2315 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java @@ -32,12 +32,11 @@ public final class JavaInputReader extends EscapeAwareReader { } @Override - protected int gobbleMaxWithoutEscape(final int bufpos, final int maxReadahead) throws IOException { - int off = bufpos; - int max = min(bufpos + maxReadahead, input.length()); + protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { + int off = this.bufpos; boolean noBackSlash = false; int notEscapeEnd = this.savedNotEscapeSpecialEnd; - while (off < max && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { + while (off < maxOff && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { off++; } @@ -59,12 +58,10 @@ public final class JavaInputReader extends EscapeAwareReader { replaceFirstBackslashWithEscape(firstBslashOff, off); this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - this.bufpos = off + 5; - this.recordEscape(firstBslashOff, off + 5 - firstBslashOff); - return firstBslashOff + 1; + return recordEscape(firstBslashOff, off + 5 - firstBslashOff, 1); } else { // not an escape sequence - int min = min(bufpos + maxReadahead, off); + int min = min(maxOff, off); // save the number of backslashes that are part of the escape, // might have been cut in half by the maxReadahead this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java index 0f03a7bed9..59ef30a854 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java @@ -85,7 +85,7 @@ public class JavaInputReaderTest { Assert.assertEquals(1, read); assertBufferIsJust("abc\\\\", chars, 0); - read = r.read(chars, 5, 8); + read = r.read(chars, 5, chars.length - 5); Assert.assertEquals(5, read); assertBufferIsJust("abc\\\\\\dede", chars, 0); @@ -125,7 +125,7 @@ public class JavaInputReaderTest { Assert.assertEquals(9, read); assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); - read = r.read(chars, 9, 10); + read = r.read(chars, 9, chars.length - 9); Assert.assertEquals(-1, read); assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java index 378dfb1011..4b3f3417b7 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -23,35 +23,30 @@ public class CppEscapeReader extends EscapeAwareReader { } @Override - protected int gobbleMaxWithoutEscape(int bufpos, int maxReadahead) throws IOException { - int off = bufpos; - int max = min(bufpos + maxReadahead, input.length()); + protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { + int off = this.bufpos; boolean noBackSlash = false; int notEscapeEnd = this.savedNotEscapeSpecialEnd; - while (off < max && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { + while (off < maxOff && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { off++; } - if (noBackSlash) { + if (noBackSlash || off == maxOff) { this.bufpos = off; return off; } final int backSlackOff = off++; if (input.charAt(off) == NEWLINE) { - recordEscape(backSlackOff, 2); - this.bufpos = off + 2; - return backSlackOff; + return recordEscape(backSlackOff, 2, 0); } else if (input.charAt(off) == CARRIAGE_RETURN) { if (input.charAt(++off) == NEWLINE) { - recordEscape(backSlackOff, 3); - this.bufpos = off + 3; - return backSlackOff; + return recordEscape(backSlackOff, 3, 0); } } // not an escape sequence - int min = min(bufpos + maxReadahead, off); + int min = min(maxOff, off); // save the number of backslashes that are part of the escape, // might have been cut in half by the maxReadahead this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 9d6df7dd49..a8823f011f 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -18,7 +18,7 @@ import net.sourceforge.pmd.util.document.TextDocument; public class CppCharStreamTest { @NonNull - public CharStream charStreamFor(String source) { + public CharStream charStreamFor(String source) throws IOException { return NewCharStream.open(new CPPTokenizer().newTokenDoc(TextDocument.readOnlyString(source))); } From 3b309ce628c4bccdb166e0ddf5385d5cc990f19c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 10:20:10 +0200 Subject: [PATCH 007/136] Gobble multiple us --- .../pmd/lang/ast/impl/io/JavaInputReader.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java index 53da0c2315..727e5e333c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java @@ -51,11 +51,14 @@ public final class JavaInputReader extends EscapeAwareReader { } int bslashCount = off - firstBslashOff; - // this condition is "is there an escape at offset firstBslashOff" - if ((bslashCount & 1) == 1 // odd number of backslashes - && off < input.length() - 4 // at least 5 chars to form the escape ('u' + 4 hex digits) - && input.charAt(off) == 'u') { // the char after the last backslash is a 'u' - + // is there an escape at offset firstBslashOff? + if ((bslashCount & 1) == 1 // odd number of backslashes + && off < input.length() && input.charAt(off) == 'u') { // at least one 'u' + // odd number of backslashes, this is enough to expect an escape or throw an exception + while (off < input.length() && input.charAt(off) == 'u') { + // consume all the 'u's + off++; + } replaceFirstBackslashWithEscape(firstBslashOff, off); this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; return recordEscape(firstBslashOff, off + 5 - firstBslashOff, 1); @@ -79,8 +82,8 @@ public final class JavaInputReader extends EscapeAwareReader { | hexVal(input.charAt(++offOfTheU)) ); input.set(posOfFirstBackSlash, c); // replace the start char of the backslash - } catch (NumberFormatException e) { + } catch (NumberFormatException | IndexOutOfBoundsException e) { String message = "Invalid escape sequence at line " + getLine(posOfFirstBackSlash) + ", column " + getColumn(posOfFirstBackSlash); From 052103ff0c2bfdb38bb56cb7676161e1aca6576f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 10:56:29 +0200 Subject: [PATCH 008/136] Test several escapes --- .../pmd/lang/ast/impl/io/EscapeTracker.java | 2 +- .../pmd/lang/ast/impl/io/JavaInputReader.java | 8 ++++---- .../lang/ast/impl/io/JavaInputReaderTest.java | 20 +++++++++++++++++++ 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java index 7b3a23798c..1c5fb513ca 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java @@ -181,7 +181,7 @@ class EscapeTracker { pos -= numChars; // then there were no escapes before the 'pos' } else { int inoff = pos; - for (int i = nextEscape - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { + for (int i = maxEscape() - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { int esc = inOff(i); if (esc == inoff) { inoff -= inLen(i); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java index 727e5e333c..7e3eb6f087 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java @@ -59,9 +59,9 @@ public final class JavaInputReader extends EscapeAwareReader { // consume all the 'u's off++; } - replaceFirstBackslashWithEscape(firstBslashOff, off); + int end = replaceFirstBackslashWithEscape(firstBslashOff, off - 1); this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - return recordEscape(firstBslashOff, off + 5 - firstBslashOff, 1); + return recordEscape(firstBslashOff, end - firstBslashOff, 1); } else { // not an escape sequence int min = min(maxOff, off); @@ -73,7 +73,7 @@ public final class JavaInputReader extends EscapeAwareReader { } } - private void replaceFirstBackslashWithEscape(int posOfFirstBackSlash, int offOfTheU) throws IOException { + private int replaceFirstBackslashWithEscape(int posOfFirstBackSlash, int offOfTheU) throws IOException { try { char c = (char) ( hexVal(input.charAt(++offOfTheU)) << 12 @@ -82,7 +82,7 @@ public final class JavaInputReader extends EscapeAwareReader { | hexVal(input.charAt(++offOfTheU)) ); input.set(posOfFirstBackSlash, c); // replace the start char of the backslash - + return offOfTheU + 1; } catch (NumberFormatException | IndexOutOfBoundsException e) { String message = "Invalid escape sequence at line " + getLine(posOfFirstBackSlash) + ", column " diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java index 59ef30a854..3169ed9bde 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java @@ -112,6 +112,26 @@ public class JavaInputReaderTest { } } + @Test + public void testSeveralEscapes() throws IOException { + + String input = "abc\\\\\\u00a0d\\uu00a0ede"; + try (JavaInputReader r = readString(input)) { + + char[] chars = new char[20]; + + int read = r.read(chars, 0, 5); + + Assert.assertEquals(5, read); + assertBufferIsJust("abc\u00a0d", chars, 0); + + read = r.read(chars, 5, 4); + + Assert.assertEquals(4, read); + assertBufferIsJust("abc\u00a0d\u00a0ede", chars, 0); + } + } + @Test public void testAnEscapeInsideBlock() throws IOException { From d3de455ede2abe31bc320a08c725ad1d168f96a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 12:31:11 +0200 Subject: [PATCH 009/136] Fix backup --- .../pmd/lang/ast/impl/io/EscapeTracker.java | 56 ++++++++++++------ .../ast/impl/javacc/JjtreeParserAdapter.java | 2 +- .../ast/impl/javacc/CharStreamImplTest.java | 57 ++++++++++++++++--- 3 files changed, 88 insertions(+), 27 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java index 1c5fb513ca..5c9bd03921 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.lang.ast.impl.io; +import static java.lang.Integer.max; + import java.io.EOFException; import net.sourceforge.pmd.util.document.Chars; @@ -27,11 +29,10 @@ class EscapeTracker { * * Eg for "a\u00a0b" (translates as "a b"), the buffer looks like * [a u00a0b] - * ^ this char has been replaced with the translated value of the escape - * ^^^^^ these characters are only present in the input, we jump over them when reading - * ^ off - * ^ invalid - * ^ off + len + * ^ (off) this char has been replaced with the translated value of the escape + * ^^^^^ (off + len - invalid) these characters are only present in the input, we jump over them when reading + * ^ (invalid) + * ^ (off + len) * * The escape record is (1,6,2) * @@ -66,17 +67,24 @@ class EscapeTracker { } private int inOff(int idx) { + assert idx < nextFreeIdx; return escapeRecords[idx]; } private int inLen(int idx) { + assert idx < nextFreeIdx; return escapeRecords[idx + 1]; } private int invalidIdx(int idx) { + assert idx < nextFreeIdx; return escapeRecords[idx + 2]; } + private int indexAfter(int idx) { + return inOff(idx) + inLen(idx); + } + /** * Convert an offset in the translated file into an offset in * the untranslated input. @@ -157,8 +165,9 @@ class EscapeTracker { char c; if (nextEscape < maxEscape() && pos == invalidIdx(nextEscape)) { - pos += inLen(nextEscape); // add escape length + int pos = indexAfter(nextEscape); // jump past escape c = buf.charAt(pos); + this.pos = pos + 1; this.nextEscape += RECORD_SIZE; } else { c = buf.charAt(pos); @@ -168,6 +177,7 @@ class EscapeTracker { return c; } + void backup(int numChars) { ensureMarked(); if (numChars > markLength()) { @@ -180,20 +190,30 @@ class EscapeTracker { if (nextEscape <= 0) { pos -= numChars; // then there were no escapes before the 'pos' } else { - int inoff = pos; - for (int i = maxEscape() - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { - int esc = inOff(i); - if (esc == inoff) { - inoff -= inLen(i); - } else if (esc > inoff) { - // then the current escape was before what we're looking at + int newOff = pos; + for (int i = nextEscape - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { + // aa __|||bb + // ^ invalid + // ^^^ jumped + // ^ inOff + // ^^ translated + + int nc = numChars; + numChars -= newOff - indexAfter(i); + newOff = max(indexAfter(i), newOff - nc); + if (numChars <= 0) { // skip "bb", ie everything after the escape break; - } else { - inoff--; } + + newOff = invalidIdx(i) - 1; // jump back over the escape ||| numChars--; + nextEscape = i; + + if (numChars <= 0) { + break; + } } - pos = inoff - numChars; + pos = newOff - numChars; // numChars is the remainder } } @@ -218,11 +238,11 @@ class EscapeTracker { int esc = markEscape; while (cur < pos && esc < nextEscape) { sb.append(buf, cur, invalidIdx(esc)); - cur = inOff(esc) + inLen(esc); + cur = indexAfter(esc); esc += RECORD_SIZE; } // no more escape in the range, append everything until the pos - sb.append(buf, cur, pos + 1); + sb.append(buf, cur, pos); assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength(); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index 9efb6dc342..e7625b446b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -37,7 +37,7 @@ public abstract class JjtreeParserAdapter implements Parser CharStream charStream = NewCharStream.open(doc); return parseImpl(charStream, task); } catch (IOException e) { - throw new TokenMgrError(-1, -1, fileName, "IO error", e); + throw new TokenMgrError(-1, -1, task.getFileDisplayName(), "IO error", e); } catch (TokenMgrError tme) { throw tme.setFileName(task.getFileDisplayName()); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java index 5eec3ba4d9..5e88930067 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java @@ -15,7 +15,10 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; +import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; public class CharStreamImplTest { @@ -26,7 +29,7 @@ public class CharStreamImplTest { @Test public void testReadZeroChars() throws IOException { - CharStream stream = getCharStream(""); + CharStream stream = simpleCharStream(""); expect.expect(EOFException.class); @@ -42,7 +45,7 @@ public class CharStreamImplTest { @Test public void testReadEofChars() throws IOException { - CharStream stream = getCharStream(""); + CharStream stream = simpleCharStream(""); expect.expect(EOFException.class); @@ -58,7 +61,7 @@ public class CharStreamImplTest { @Test public void testMultipleEofReads() throws IOException { - CharStream stream = getCharStream(""); + CharStream stream = simpleCharStream(""); for (int i = 0; i < 3; i++) { try { @@ -74,7 +77,7 @@ public class CharStreamImplTest { @Test public void testReadStuff() throws IOException { - CharStream stream = getCharStream("abcd"); + CharStream stream = simpleCharStream("abcd"); assertEquals('a', stream.readChar()); assertEquals('b', stream.readChar()); @@ -88,7 +91,7 @@ public class CharStreamImplTest { @Test public void testReadBacktrack() throws IOException { - CharStream stream = getCharStream("abcd"); + CharStream stream = simpleCharStream("abcd"); assertEquals('a', stream.BeginToken()); assertEquals('b', stream.readChar()); @@ -106,14 +109,52 @@ public class CharStreamImplTest { stream.readChar(); } - public CharStream getCharStream(String abcd) throws IOException { + @Test + public void testReadBacktrackWithEscapes() throws IOException { + + CharStream stream = javaCharStream("__\\u00a0_\\u00a0_"); + + assertEquals('_', stream.BeginToken()); + assertEquals('_', stream.readChar()); + assertEquals('\u00a0', stream.readChar()); + assertEquals('_', stream.readChar()); + + assertEquals("__\u00a0_", stream.GetImage()); + + stream.backup(2); + assertEquals('\u00a0', stream.readChar()); + assertEquals('_', stream.readChar()); + assertEquals('\u00a0', stream.readChar()); + + assertEquals("__\u00a0_\u00a0", stream.GetImage()); + assertEquals('_', stream.readChar()); + stream.backup(2); + assertEquals('\u00a0', stream.BeginToken()); + assertEquals('_', stream.readChar()); + + assertEquals("\u00a0_", stream.GetImage()); + + expect.expect(EOFException.class); + stream.readChar(); + } + + public static CharStream simpleCharStream(String abcd) throws IOException { return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd))); } + public static CharStream javaCharStream(String abcd) throws IOException { + return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)) { + @Override + public EscapeAwareReader newReader(Chars text) { + return new JavaInputReader(text); + } + }); + } + @Test public void testBacktrackTooMuch() throws IOException { - CharStream stream = getCharStream("abcd"); + CharStream stream = simpleCharStream("abcd"); assertEquals('a', stream.readChar()); assertEquals('b', stream.readChar()); @@ -127,7 +168,7 @@ public class CharStreamImplTest { @Test public void testBacktrackTooMuch2() throws IOException { - CharStream stream = getCharStream("abcd"); + CharStream stream = simpleCharStream("abcd"); assertEquals('a', stream.BeginToken()); assertEquals('b', stream.readChar()); From 95bac5eb950c8859a1805eefc2673d2adf021add Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 12:50:34 +0200 Subject: [PATCH 010/136] Convert CharStream to class --- .../pmd/cpd/internal/JavaCCTokenizer.java | 5 +- .../pmd/lang/ast/impl/io/NewCharStream.java | 84 ------------------- .../ast/{ => impl/javacc}/CharStream.java | 84 +++++++++++++------ .../{io => javacc}/EscapeAwareReader.java | 6 +- .../impl/{io => javacc}/EscapeTracker.java | 21 +++-- .../impl/{io => javacc}/JavaInputReader.java | 6 +- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 1 - .../ast/impl/javacc/JavaccTokenDocument.java | 2 - .../ast/impl/javacc/JjtreeParserAdapter.java | 4 +- .../lang/ast/impl/io/JavaInputReaderTest.java | 1 + .../ast/impl/javacc/CharStreamImplTest.java | 8 +- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 4 +- .../pmd/lang/cpp/ast/CppEscapeReader.java | 2 +- .../pmd/cpd/CppCharStreamTest.java | 5 +- pmd-java/etc/grammar/Java.jjt | 2 +- .../sourceforge/pmd/cpd/JavaTokenizer.java | 2 +- .../pmd/lang/java/ast/JavaParser.java | 2 +- .../pmd/lang/java/ast/JavaTokenDocument.java | 6 +- .../pmd/cpd/EcmascriptTokenizer.java | 2 +- pmd-jsp/etc/grammar/Jsp.jjt | 2 +- .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 6 +- .../pmd/lang/jsp/ast/JspParser.java | 2 +- .../sourceforge/pmd/cpd/MatlabTokenizer.java | 2 +- .../pmd/cpd/ModelicaTokenizer.java | 2 +- .../pmd/lang/modelica/ast/ModelicaParser.java | 2 +- .../pmd/cpd/ObjectiveCTokenizer.java | 2 +- pmd-plsql/etc/grammar/PLSQL.jjt | 2 +- .../sourceforge/pmd/cpd/PLSQLTokenizer.java | 2 +- .../pmd/lang/plsql/ast/PLSQLParser.java | 2 +- .../sourceforge/pmd/cpd/PythonTokenizer.java | 2 +- pmd-visualforce/etc/grammar/Vf.jjt | 2 +- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 6 +- .../sourceforge/pmd/lang/vf/ast/VfParser.java | 2 +- pmd-vm/etc/grammar/Vm.jjt | 2 +- .../sourceforge/pmd/lang/vm/ast/VmParser.java | 2 +- 35 files changed, 118 insertions(+), 169 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/{ => impl/javacc}/CharStream.java (60%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/{io => javacc}/EscapeAwareReader.java (97%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/{io => javacc}/EscapeTracker.java (95%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/{io => javacc}/JavaInputReader.java (96%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 5e3c06a735..7f6b906172 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -13,9 +13,8 @@ import net.sourceforge.pmd.cpd.Tokens; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.TextDocument; @@ -27,7 +26,7 @@ public abstract class JavaCCTokenizer implements Tokenizer { protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { TextDocument textDocument = TextDocument.create(TextFile.cpdCompat(sourceCode)); JavaccTokenDocument tokenDoc = newTokenDoc(textDocument); - return makeLexerImpl(NewCharStream.open(tokenDoc)); + return makeLexerImpl(CharStream.create(tokenDoc)); } protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java deleted file mode 100644 index dd8ded5daf..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/NewCharStream.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.io; - -import java.io.EOFException; -import java.io.IOException; - -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.util.document.FileLocation; -import net.sourceforge.pmd.util.document.TextDocument; - -public class NewCharStream implements CharStream { - - private final JavaccTokenDocument document; - private final EscapeTracker.Cursor cursor; - - private NewCharStream(JavaccTokenDocument document, EscapeTracker.Cursor cursor) { - this.document = document; - this.cursor = cursor; - } - - public static CharStream open(JavaccTokenDocument doc) throws IOException { - try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { - reader.translate(); - return new NewCharStream(doc, reader.escapes.new Cursor(reader.input)); - } - } - - @Override - public JavaccTokenDocument getTokenDocument() { - return document; - } - - @Override - public char readChar() throws EOFException { - return cursor.next(); - } - - @Override - public char BeginToken() throws EOFException { - cursor.mark(); - return cursor.next(); - } - - @Override - public String GetImage() { - StringBuilder sb = new StringBuilder(); - cursor.markToString(sb); - return sb.toString(); - } - - @Override - public void backup(int amount) { - cursor.backup(amount); - } - - @Override - public int getEndColumn() { - return endLocation().getEndColumn(); - } - - @Override - public int getEndLine() { - return endLocation().getEndLine(); - } - - private FileLocation endLocation() { - TextDocument textDoc = document.getTextDocument(); - return textDoc.toLocation(textDoc.createRegion(getEndOffset(), 0)); - } - - @Override - public int getStartOffset() { - return cursor.markOutOffset(); - } - - @Override - public int getEndOffset() { - return cursor.curOutOffset(); - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java similarity index 60% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index 4565dafc42..a81d2af44a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -2,13 +2,14 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast; +package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.EOFException; import java.io.IOException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.util.document.FileLocation; +import net.sourceforge.pmd.util.document.TextDocument; /** * PMD flavour of character streams used by JavaCC parsers. @@ -17,7 +18,25 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; * * rename methods to match decent naming conventions * * move to impl.javacc package */ -public interface CharStream { +public final class CharStream { + + private final JavaccTokenDocument tokenDoc; + private final EscapeTracker.Cursor cursor; + + private CharStream(JavaccTokenDocument tokenDoc, EscapeTracker.Cursor cursor) { + this.tokenDoc = tokenDoc; + this.cursor = cursor; + } + + /** + * Create a new char stream for the given document. + */ + public static CharStream create(JavaccTokenDocument doc) throws IOException { + try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { + reader.translate(); + return new CharStream(doc, reader.escapes.new Cursor(reader.input)); + } + } /** * Returns the next character from the input. After a {@link #backup(int)}, @@ -26,9 +45,10 @@ public interface CharStream { * @return The next character * * @throws EOFException Upon EOF - * @throws IOException If the underlying char stream throws EOF */ - char readChar() throws IOException; + public char readChar() throws EOFException { + return cursor.next(); + } /** @@ -37,14 +57,21 @@ public interface CharStream { * the buffer between two successive calls to this method to implement * backup correctly. */ - char BeginToken() throws IOException; + public char BeginToken() throws EOFException { + cursor.mark(); + return cursor.next(); + } /** * Returns a string made up of characters from the token mark up to * to the current buffer position. */ - String GetImage(); + public String GetImage() { + StringBuilder sb = new StringBuilder(); + cursor.markToString(sb); + return sb.toString(); + } /** @@ -65,13 +92,13 @@ public interface CharStream { * @throws IndexOutOfBoundsException If len is greater than the length of the * current token */ - default char[] GetSuffix(int len) { + public char[] GetSuffix(int len) { String t = GetImage(); return t.substring(t.length() - len).toCharArray(); } - default void appendSuffix(StringBuilder sb, int len) { + public void appendSuffix(StringBuilder sb, int len) { String t = GetImage(); sb.append(t, t.length() - len, t.length()); } @@ -89,41 +116,46 @@ public interface CharStream { * @throws AssertionError If the requested amount is greater than the * number of read chars */ - void backup(int amount); - + public void backup(int amount) { + cursor.backup(amount); + } /** Returns the column number of the last character for the current token. */ - int getEndColumn(); - - - /** Returns the line number of the last character for current token. */ - int getEndLine(); - - - default int getBeginColumn() { - return -1; + public int getEndColumn() { + return endLocation().getEndColumn(); } - default int getBeginLine() { - return -1; + /** Returns the line number of the last character for current token. */ + public int getEndLine() { + return endLocation().getEndLine(); + } + + + private FileLocation endLocation() { + TextDocument textDoc = tokenDoc.getTextDocument(); + return textDoc.toLocation(textDoc.createRegion(getEndOffset(), 0)); } /** Returns the start offset of the current token (in the original source), inclusive. */ - int getStartOffset(); + public int getStartOffset() { + return cursor.markOutOffset(); + } /** Returns the end offset of the current token (in the original source), exclusive. */ - int getEndOffset(); + public int getEndOffset() { + return cursor.curOutOffset(); + } /** * Returns the token document for the tokens being built. Having it * here is the most convenient place for the time being. */ - default JavaccTokenDocument getTokenDocument() { - return null; // for VelocityCharStream + public JavaccTokenDocument getTokenDocument() { + return tokenDoc; } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java similarity index 97% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java index fa8412e920..8b0f8eb0e1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java @@ -2,7 +2,11 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.io; +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc; import static java.lang.Integer.min; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java similarity index 95% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java index 5c9bd03921..9745741c6c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java @@ -2,7 +2,11 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.io; +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc; import static java.lang.Integer.max; @@ -122,7 +126,6 @@ class EscapeTracker { /** Backend for a CharStream. */ class Cursor { - /** * This is the index in buf of the next char to read, it always * holds that buf[pos] is a valid character. @@ -158,7 +161,7 @@ class EscapeTracker { this.buf = buf; } - char next() throws EOFException { + public char next() throws EOFException { if (pos == buf.length()) { throw new EOFException(); } @@ -178,7 +181,7 @@ class EscapeTracker { } - void backup(int numChars) { + public void backup(int numChars) { ensureMarked(); if (numChars > markLength()) { throw new IllegalArgumentException( @@ -217,13 +220,13 @@ class EscapeTracker { } } - void mark() { + public void mark() { this.mark = pos; this.markEscape = nextEscape; this.markOutOffset = outOffset; } - void markToString(StringBuilder sb) { + public void markToString(StringBuilder sb) { ensureMarked(); int prevLength = sb.length(); @@ -256,15 +259,15 @@ class EscapeTracker { assert markEscape <= escapeRecords.length : "Wrong escape mark"; } - int curOutOffset() { + public int curOutOffset() { return outOffset; } - int markOutOffset() { + public int markOutOffset() { return markOutOffset; } - int markLength() { + public int markLength() { return outOffset - markOutOffset; } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java similarity index 96% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java index 7e3eb6f087..68f1970809 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java @@ -2,7 +2,11 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.io; +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc; import static java.lang.Integer.min; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index b57889ff55..d2e0c02045 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -4,7 +4,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextRegion; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 8c0e82ce4a..896193d356 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -7,9 +7,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index e7625b446b..c37da63d78 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -7,11 +7,9 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.IOException; import net.sourceforge.pmd.lang.Parser; -import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.RootNode; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -34,7 +32,7 @@ public abstract class JjtreeParserAdapter implements Parser JavaccTokenDocument doc = newDocumentImpl(task.getTextDocument()); try { - CharStream charStream = NewCharStream.open(doc); + CharStream charStream = CharStream.create(doc); return parseImpl(charStream, task); } catch (IOException e) { throw new TokenMgrError(-1, -1, task.getFileDisplayName(), "IO error", e); diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java index 3169ed9bde..61c2de1725 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java @@ -10,6 +10,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; import net.sourceforge.pmd.util.document.Chars; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java index 5e88930067..816d184e42 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java @@ -14,10 +14,6 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; -import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -139,11 +135,11 @@ public class CharStreamImplTest { } public static CharStream simpleCharStream(String abcd) throws IOException { - return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd))); + return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd))); } public static CharStream javaCharStream(String abcd) throws IOException { - return NewCharStream.open(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)) { + return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)) { @Override public EscapeAwareReader newReader(Chars text) { return new JavaInputReader(text); diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 6aa98771b6..9d0bcb4a93 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -13,8 +13,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java index 4b3f3417b7..99f4581f5e 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -8,7 +8,7 @@ import static java.lang.Integer.min; import java.io.IOException; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; import net.sourceforge.pmd.util.document.Chars; public class CppEscapeReader extends EscapeAwareReader { diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index a8823f011f..fab00aab81 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -11,15 +11,14 @@ import java.io.IOException; import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.NewCharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.document.TextDocument; public class CppCharStreamTest { @NonNull public CharStream charStreamFor(String source) throws IOException { - return NewCharStream.open(new CPPTokenizer().newTokenDoc(TextDocument.readOnlyString(source))); + return CharStream.create(new CPPTokenizer().newTokenDoc(TextDocument.readOnlyString(source))); } @Test diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index 5fcf6d995d..b2eb1fbba4 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -240,7 +240,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Set; import java.util.Map; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.Node; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index 83c69ae100..aa304b6ab2 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -13,7 +13,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index a3bf759260..ea8a1b2b1b 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -4,7 +4,7 @@ package net.sourceforge.pmd.lang.java.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index a3a67d2fb0..a1d74fe154 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -14,9 +14,9 @@ import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.WHITESPACE; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index 1294536009..d66d74949f 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5TokenKinds; diff --git a/pmd-jsp/etc/grammar/Jsp.jjt b/pmd-jsp/etc/grammar/Jsp.jjt index c5400e8b6c..9e4483a441 100644 --- a/pmd-jsp/etc/grammar/Jsp.jjt +++ b/pmd-jsp/etc/grammar/Jsp.jjt @@ -30,7 +30,7 @@ options { PARSER_BEGIN(JspParserImpl) package net.sourceforge.pmd.lang.jsp.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index fcb65f40f2..48a62f5131 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -6,9 +6,9 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java index da368c871b..6d8cf7f3a0 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.jsp.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java index 3273203072..9459c44696 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.matlab.ast.MatlabTokenKinds; diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java index 71f1bb930f..3258a3cda7 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.modelica.ast.ModelicaTokenKinds; diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java index cda61a0ba1..058941d176 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java @@ -4,7 +4,7 @@ package net.sourceforge.pmd.lang.modelica.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java index e101af905d..acccfcd24a 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.objectivec.ast.ObjectiveCTokenKinds; diff --git a/pmd-plsql/etc/grammar/PLSQL.jjt b/pmd-plsql/etc/grammar/PLSQL.jjt index 097b6391ff..090f90e235 100644 --- a/pmd-plsql/etc/grammar/PLSQL.jjt +++ b/pmd-plsql/etc/grammar/PLSQL.jjt @@ -154,7 +154,7 @@ package net.sourceforge.pmd.lang.plsql.ast; import java.io.*; import net.sourceforge.pmd.lang.ast.Node; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class PLSQLParserImpl { diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index f302bc540e..77abbf8794 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -8,7 +8,7 @@ import java.util.Properties; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.plsql.ast.PLSQLTokenKinds; diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java index ed2cb3e08d..5cf6fa5ef7 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.plsql.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 9e61ea2067..70755980ea 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -10,7 +10,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds; diff --git a/pmd-visualforce/etc/grammar/Vf.jjt b/pmd-visualforce/etc/grammar/Vf.jjt index 8edfcc2612..2af403e0cb 100644 --- a/pmd-visualforce/etc/grammar/Vf.jjt +++ b/pmd-visualforce/etc/grammar/Vf.jjt @@ -13,7 +13,7 @@ options { PARSER_BEGIN(VfParserImpl) package net.sourceforge.pmd.lang.vf.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class VfParserImpl { diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 8fa9212144..ba239ef3cf 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -6,9 +6,9 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.CharStream; -import net.sourceforge.pmd.lang.ast.impl.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.io.JavaInputReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java index 5b5ff1fa69..b740cc8c63 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.vf.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-vm/etc/grammar/Vm.jjt b/pmd-vm/etc/grammar/Vm.jjt index 0106fcd6d8..678ed8de31 100644 --- a/pmd-vm/etc/grammar/Vm.jjt +++ b/pmd-vm/etc/grammar/Vm.jjt @@ -45,8 +45,8 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; -import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java index 236abc5886..a2995685d2 100644 --- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java +++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.vm.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; From c088a6add5fe1b8ee609df410d6f6ac8252b2adc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 13:02:54 +0200 Subject: [PATCH 011/136] Rename charStream methods --- javacc-wrapper.xml | 21 +++++++++++++- .../pmd/lang/ast/impl/javacc/CharStream.java | 29 +++++-------------- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 2 +- .../ast/impl/javacc/JavaccTokenDocument.java | 4 +-- .../ast/impl/javacc/CharStreamImplTest.java | 18 ++++++------ .../pmd/cpd/CppCharStreamTest.java | 8 +++-- .../sourceforge/pmd/lang/vm/ast/VmParser.java | 2 +- 7 files changed, 45 insertions(+), 39 deletions(-) diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index 5181f10667..5392693a56 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -60,7 +60,7 @@ - + @@ -276,6 +276,25 @@ + + + + + + + + + + + + + + + + + + + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index a81d2af44a..d9be04d0a5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -57,7 +57,7 @@ public final class CharStream { * the buffer between two successive calls to this method to implement * backup correctly. */ - public char BeginToken() throws EOFException { + public char markTokenStart() throws EOFException { cursor.mark(); return cursor.next(); } @@ -67,7 +67,7 @@ public final class CharStream { * Returns a string made up of characters from the token mark up to * to the current buffer position. */ - public String GetImage() { + public String getTokenImage() { StringBuilder sb = new StringBuilder(); cursor.markToString(sb); return sb.toString(); @@ -75,31 +75,16 @@ public final class CharStream { /** - * Returns an array of characters that make up the suffix of length 'len' for - * the current token. This is used to build up the matched string - * for use in actions in the case of MORE. A simple and inefficient - * implementation of this is as follows : - * - * 

{@code
-     * String t = tokenImage();
-     * return t.substring(t.length() - len).toCharArray();
-     * }
+ * Appends the suffix of length 'len' of the current token to the given + * string builder. This is used to build up the matched string + * for use in actions in the case of MORE. * * @param len Length of the returned array * - * @return The suffix - * - * @throws IndexOutOfBoundsException If len is greater than the length of the - * current token + * @throws IndexOutOfBoundsException If len is greater than the length of the current token */ - public char[] GetSuffix(int len) { - String t = GetImage(); - return t.substring(t.length() - len).toCharArray(); - } - - public void appendSuffix(StringBuilder sb, int len) { - String t = GetImage(); + String t = getTokenImage(); sb.append(t, t.length() - len, t.length()); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index d2e0c02045..fb5195f4d6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -178,7 +178,7 @@ public class JavaccToken implements GenericToken { public JavaccToken replaceImage(CharStream charStream) { return new JavaccToken( this.kind, - charStream.GetImage(), + charStream.getTokenImage(), this.startOffset, charStream.getEndOffset(), this.document diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 896193d356..27cb3bc609 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -102,7 +102,7 @@ public class JavaccTokenDocument extends TokenDocument { * @param cs Char stream of the file. This can be used to get text * coordinates and the image * @param image Shared instance of the image token. If this is non-null, - * then no call to {@link CharStream#GetImage()} should be + * then no call to {@link CharStream#getTokenImage()} should be * issued. * * @return A new token @@ -110,7 +110,7 @@ public class JavaccTokenDocument extends TokenDocument { public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { return new JavaccToken( kind, - image == null ? cs.GetImage() : image, + image == null ? cs.getTokenImage() : image, cs.getStartOffset(), cs.getEndOffset(), this diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java index 816d184e42..2d57cae381 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java @@ -89,12 +89,12 @@ public class CharStreamImplTest { CharStream stream = simpleCharStream("abcd"); - assertEquals('a', stream.BeginToken()); + assertEquals('a', stream.markTokenStart()); assertEquals('b', stream.readChar()); assertEquals('c', stream.readChar()); assertEquals('d', stream.readChar()); - assertEquals("abcd", stream.GetImage()); + assertEquals("abcd", stream.getTokenImage()); stream.backup(2); assertEquals('c', stream.readChar()); @@ -110,25 +110,25 @@ public class CharStreamImplTest { CharStream stream = javaCharStream("__\\u00a0_\\u00a0_"); - assertEquals('_', stream.BeginToken()); + assertEquals('_', stream.markTokenStart()); assertEquals('_', stream.readChar()); assertEquals('\u00a0', stream.readChar()); assertEquals('_', stream.readChar()); - assertEquals("__\u00a0_", stream.GetImage()); + assertEquals("__\u00a0_", stream.getTokenImage()); stream.backup(2); assertEquals('\u00a0', stream.readChar()); assertEquals('_', stream.readChar()); assertEquals('\u00a0', stream.readChar()); - assertEquals("__\u00a0_\u00a0", stream.GetImage()); + assertEquals("__\u00a0_\u00a0", stream.getTokenImage()); assertEquals('_', stream.readChar()); stream.backup(2); - assertEquals('\u00a0', stream.BeginToken()); + assertEquals('\u00a0', stream.markTokenStart()); assertEquals('_', stream.readChar()); - assertEquals("\u00a0_", stream.GetImage()); + assertEquals("\u00a0_", stream.getTokenImage()); expect.expect(EOFException.class); stream.readChar(); @@ -154,7 +154,7 @@ public class CharStreamImplTest { assertEquals('a', stream.readChar()); assertEquals('b', stream.readChar()); - assertEquals('c', stream.BeginToken()); + assertEquals('c', stream.markTokenStart()); assertEquals('d', stream.readChar()); expect.expect(IllegalArgumentException.class); @@ -166,7 +166,7 @@ public class CharStreamImplTest { CharStream stream = simpleCharStream("abcd"); - assertEquals('a', stream.BeginToken()); + assertEquals('a', stream.markTokenStart()); assertEquals('b', stream.readChar()); assertEquals('c', stream.readChar()); assertEquals('d', stream.readChar()); diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index fab00aab81..55c4aef643 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -42,13 +42,15 @@ public class CppCharStreamTest { } private void assertStream(CharStream stream, String token) throws IOException { - char c = stream.BeginToken(); + char c = stream.markTokenStart(); assertEquals(token.charAt(0), c); for (int i = 1; i < token.length(); i++) { c = stream.readChar(); assertEquals(token + " char at " + i + ": " + token.charAt(i) + " != " + c, token.charAt(i), c); } - assertEquals(token, stream.GetImage()); - assertEquals(token, new String(stream.GetSuffix(token.length()))); + assertEquals(token, stream.getTokenImage()); + StringBuilder sb = new StringBuilder(); + stream.appendSuffix(sb, token.length()); + assertEquals(token, sb.toString()); } } diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java index a2995685d2..845653d06c 100644 --- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java +++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java @@ -42,7 +42,7 @@ public class VmParser extends JjtreeParserAdapter { @Override public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { - String realImage = image == null ? cs.GetImage() : image; + String realImage = image == null ? cs.getTokenImage() : image; if (kind == VmTokenKinds.ESCAPE_DIRECTIVE) { realImage = escapedDirective(realImage); } From 1f8c5b65b3e7ce4be59963553ed49a2c6d384ee0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 13:12:59 +0200 Subject: [PATCH 012/136] Better implementation for appendSuffix --- javacc-wrapper.xml | 2 +- .../pmd/lang/ast/impl/javacc/CharStream.java | 19 ++++---- .../lang/ast/impl/javacc/EscapeTracker.java | 46 +++++++++++++------ pmd-cpp/etc/grammar/Cpp.jj | 2 +- pmd-javascript/etc/grammar/Ecmascript5.jj | 2 +- pmd-matlab/etc/grammar/Matlab.jj | 2 +- pmd-modelica/etc/grammar/Modelica.jjt | 2 +- pmd-objectivec/etc/grammar/ObjectiveC.jj | 2 +- pmd-python/etc/grammar/Python.jj | 2 +- 9 files changed, 49 insertions(+), 30 deletions(-) diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index 5392693a56..b9cf9b1ba1 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -407,7 +407,7 @@ public final class ${token-constants-name} \{${line.separator} * be used as a basis for a CPD Tokenizer. */ @net.sourceforge.pmd.annotation.InternalApi - public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.CharStream cs) { + public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.CharStream cs) { return new %%%TOKEN_MGR_NAME%%%(cs); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index d9be04d0a5..bc3c9606a0 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -13,10 +13,6 @@ import net.sourceforge.pmd.util.document.TextDocument; /** * PMD flavour of character streams used by JavaCC parsers. - * - * TODO for when all JavaCC languages are aligned: - * * rename methods to match decent naming conventions - * * move to impl.javacc package */ public final class CharStream { @@ -69,7 +65,7 @@ public final class CharStream { */ public String getTokenImage() { StringBuilder sb = new StringBuilder(); - cursor.markToString(sb); + cursor.appendMark(sb); return sb.toString(); } @@ -84,8 +80,7 @@ public final class CharStream { * @throws IndexOutOfBoundsException If len is greater than the length of the current token */ public void appendSuffix(StringBuilder sb, int len) { - String t = getTokenImage(); - sb.append(t, t.length() - len, t.length()); + cursor.appendMarkSuffix(sb, len); } @@ -105,13 +100,19 @@ public final class CharStream { cursor.backup(amount); } - /** Returns the column number of the last character for the current token. */ + /** + * Returns the column number of the last character for the current token. + * This is only used for parse exceptions and is very inefficient. + */ public int getEndColumn() { return endLocation().getEndColumn(); } - /** Returns the line number of the last character for current token. */ + /** + * Returns the line number of the last character for current token. + * This is only used for parse exceptions and is very inefficient. + */ public int getEndLine() { return endLocation().getEndLine(); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java index 9745741c6c..a8e9f84d17 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java @@ -226,30 +226,48 @@ class EscapeTracker { this.markOutOffset = outOffset; } - public void markToString(StringBuilder sb) { + public void appendMarkSuffix(StringBuilder sb, int suffixLen) { ensureMarked(); + assert suffixLen <= markLength(); - int prevLength = sb.length(); if (markEscape == nextEscape) { // no escape in the marked range - sb.append(buf, mark, pos); + sb.append(buf, pos - suffixLen, pos); } else { - sb.ensureCapacity(markLength()); - - int cur = mark; - int esc = markEscape; - while (cur < pos && esc < nextEscape) { - sb.append(buf, cur, invalidIdx(esc)); - cur = indexAfter(esc); - esc += RECORD_SIZE; + if (suffixLen == markLength()) { + appendMark(sb); + } else { + // fallback inefficient implementation + StringBuilder tmp = new StringBuilder(); + appendMark(tmp); + sb.append(tmp, tmp.length() - suffixLen, tmp.length()); } - // no more escape in the range, append everything until the pos - sb.append(buf, cur, pos); - assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength(); } } + public void appendMark(StringBuilder sb) { + if (markEscape == nextEscape) { + // no escape in the marked range + sb.append(buf, mark, pos); + return; + } + + sb.ensureCapacity(markLength()); + int prevLength = sb.length(); + + int cur = mark; + int esc = markEscape; + while (cur < pos && esc < nextEscape) { + sb.append(buf, cur, invalidIdx(esc)); + cur = indexAfter(esc); + esc += RECORD_SIZE; + } + // no more escape in the range, append everything until the pos + sb.append(buf, cur, pos); + assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength(); + } + private void ensureMarked() { if (mark == Integer.MAX_VALUE) { throw new IllegalStateException("Mark is not set"); diff --git a/pmd-cpp/etc/grammar/Cpp.jj b/pmd-cpp/etc/grammar/Cpp.jj index 450f772c41..eed7cc58cf 100644 --- a/pmd-cpp/etc/grammar/Cpp.jj +++ b/pmd-cpp/etc/grammar/Cpp.jj @@ -32,7 +32,7 @@ options { PARSER_BEGIN(CppParserImpl) package net.sourceforge.pmd.lang.cpp.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public final class CppParserImpl { diff --git a/pmd-javascript/etc/grammar/Ecmascript5.jj b/pmd-javascript/etc/grammar/Ecmascript5.jj index e180c2f760..a9cf14bcbb 100644 --- a/pmd-javascript/etc/grammar/Ecmascript5.jj +++ b/pmd-javascript/etc/grammar/Ecmascript5.jj @@ -15,7 +15,7 @@ options { PARSER_BEGIN(Ecmascript5ParserImpl) package net.sourceforge.pmd.lang.ecmascript5.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class Ecmascript5ParserImpl { diff --git a/pmd-matlab/etc/grammar/Matlab.jj b/pmd-matlab/etc/grammar/Matlab.jj index 891a80f886..3ff1c8b27c 100644 --- a/pmd-matlab/etc/grammar/Matlab.jj +++ b/pmd-matlab/etc/grammar/Matlab.jj @@ -21,7 +21,7 @@ options { PARSER_BEGIN(MatlabParserImpl) package net.sourceforge.pmd.lang.matlab.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class MatlabParserImpl { diff --git a/pmd-modelica/etc/grammar/Modelica.jjt b/pmd-modelica/etc/grammar/Modelica.jjt index 67039e9d1b..e2e7d63121 100644 --- a/pmd-modelica/etc/grammar/Modelica.jjt +++ b/pmd-modelica/etc/grammar/Modelica.jjt @@ -49,7 +49,7 @@ options { PARSER_BEGIN(ModelicaParserImpl) package net.sourceforge.pmd.lang.modelica.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; class ModelicaParserImpl { diff --git a/pmd-objectivec/etc/grammar/ObjectiveC.jj b/pmd-objectivec/etc/grammar/ObjectiveC.jj index e42313b6c2..171ca581fa 100644 --- a/pmd-objectivec/etc/grammar/ObjectiveC.jj +++ b/pmd-objectivec/etc/grammar/ObjectiveC.jj @@ -21,7 +21,7 @@ package net.sourceforge.pmd.lang.objectivec.ast; import java.io.*; import java.util.*; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; /** diff --git a/pmd-python/etc/grammar/Python.jj b/pmd-python/etc/grammar/Python.jj index b9c2313b62..8087042e7c 100644 --- a/pmd-python/etc/grammar/Python.jj +++ b/pmd-python/etc/grammar/Python.jj @@ -17,7 +17,7 @@ PARSER_BEGIN(PythonParserImpl) package net.sourceforge.pmd.lang.python.ast; -import net.sourceforge.pmd.lang.ast.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class PythonParserImpl { From b662cadcc0e3d97fd33b13e6b9f6c9c00edd56c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 14:19:32 +0200 Subject: [PATCH 013/136] Some opts --- .../pmd/lang/ast/impl/javacc/CharStream.java | 13 ++++-- .../lang/ast/impl/javacc/EscapeTracker.java | 41 +++++++++---------- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 10 ++++- .../ast/impl/javacc/JavaccTokenDocument.java | 2 +- .../pmd/lang/java/ast/FormalComment.java | 5 ++- .../pmd/lang/java/ast/ParserCornersTest.java | 16 ++++++++ 6 files changed, 58 insertions(+), 29 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index bc3c9606a0..3a14835da1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -8,6 +8,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.EOFException; import java.io.IOException; +import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextDocument; @@ -64,9 +65,15 @@ public final class CharStream { * to the current buffer position. */ public String getTokenImage() { - StringBuilder sb = new StringBuilder(); - cursor.appendMark(sb); - return sb.toString(); + return getTokenImageCs().toString(); + } + + /** + * Returns a string made up of characters from the token mark up to + * to the current buffer position. + */ + public Chars getTokenImageCs() { + return cursor.getMarkImage(); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java index a8e9f84d17..4de40a65ee 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java @@ -23,6 +23,7 @@ class EscapeTracker { private static final int[] EMPTY = new int[0]; private static final int RECORD_SIZE = 3; + static final EOFException EOF = new EOFException(); /* * Offsets in the input buffer where a unicode escape occurred. @@ -70,22 +71,22 @@ class EscapeTracker { escapeRecords[nextFreeIdx++] = offsetInInput + lengthInOutput; } - private int inOff(int idx) { + int inOff(int idx) { assert idx < nextFreeIdx; return escapeRecords[idx]; } - private int inLen(int idx) { + int inLen(int idx) { assert idx < nextFreeIdx; return escapeRecords[idx + 1]; } - private int invalidIdx(int idx) { + int invalidIdx(int idx) { assert idx < nextFreeIdx; return escapeRecords[idx + 2]; } - private int indexAfter(int idx) { + int indexAfter(int idx) { return inOff(idx) + inLen(idx); } @@ -163,7 +164,7 @@ class EscapeTracker { public char next() throws EOFException { if (pos == buf.length()) { - throw new EOFException(); + throw EOF; } char c; @@ -216,7 +217,11 @@ class EscapeTracker { break; } } - pos = newOff - numChars; // numChars is the remainder + if (numChars < 0) { + pos = newOff; // newOff was already clipped + } else { + pos = newOff - numChars; // numChars is the remainder + } } } @@ -233,39 +238,33 @@ class EscapeTracker { if (markEscape == nextEscape) { // no escape in the marked range - sb.append(buf, pos - suffixLen, pos); + buf.appendChars(sb, pos - suffixLen, suffixLen); } else { - if (suffixLen == markLength()) { - appendMark(sb); - } else { - // fallback inefficient implementation - StringBuilder tmp = new StringBuilder(); - appendMark(tmp); - sb.append(tmp, tmp.length() - suffixLen, tmp.length()); - } + // fallback inefficient implementation + getMarkImage().appendChars(sb, markLength() - suffixLen, suffixLen); } } - public void appendMark(StringBuilder sb) { + public Chars getMarkImage() { if (markEscape == nextEscape) { // no escape in the marked range - sb.append(buf, mark, pos); - return; + return buf.slice(mark, markLength()); } - sb.ensureCapacity(markLength()); + StringBuilder sb = new StringBuilder(markLength()); int prevLength = sb.length(); int cur = mark; int esc = markEscape; while (cur < pos && esc < nextEscape) { - sb.append(buf, cur, invalidIdx(esc)); + buf.appendChars(sb, cur, invalidIdx(esc) - cur); cur = indexAfter(esc); esc += RECORD_SIZE; } // no more escape in the range, append everything until the pos - sb.append(buf, cur, pos); + buf.appendChars(sb, cur, pos - cur); assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength(); + return Chars.wrap(sb, true); } private void ensureMarked() { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index fb5195f4d6..0c113a3a01 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; +import org.apache.commons.lang3.StringUtils; + import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextRegion; @@ -139,6 +141,10 @@ public class JavaccToken implements GenericToken { return image; } + public boolean imageEquals(CharSequence charSequence) { + return StringUtils.equals(image, charSequence); + } + @Override public TextRegion getRegion() { return TextRegion.fromBothOffsets(startOffset, endOffset); @@ -178,14 +184,14 @@ public class JavaccToken implements GenericToken { public JavaccToken replaceImage(CharStream charStream) { return new JavaccToken( this.kind, - charStream.getTokenImage(), + charStream.getTokenImageCs(), this.startOffset, charStream.getEndOffset(), this.document ); } - public JavaccToken withImage(String image) { + public JavaccToken withImage(CharSequence image) { return new JavaccToken( this.kind, image, diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 27cb3bc609..acf616d2ad 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -110,7 +110,7 @@ public class JavaccTokenDocument extends TokenDocument { public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { return new JavaccToken( kind, - image == null ? cs.getTokenImage() : image, + image == null ? cs.getTokenImageCs() : image, cs.getStartOffset(), cs.getEndOffset(), this diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java index 27103bc54a..c232b943c0 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java @@ -19,7 +19,8 @@ public class FormalComment extends Comment { public FormalComment(JavaccToken t) { super(t); - findJavadocs(); +// findJavadocs(); + } @Override @@ -27,7 +28,7 @@ public class FormalComment extends Comment { return "FormalComment"; } - private void findJavadocs() { + private void findJavadocs(JavaccToken t) { List kids = new ArrayList<>(); Matcher javadocTagMatcher = JAVADOC_TAG.matcher(getFilteredComment()); diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java index 2317d4830e..1180d2645b 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java @@ -83,6 +83,22 @@ public class ParserCornersTest extends BaseJavaTreeDumpTest { + "}"); } + @Test + public void testUnicodeEscapes() { + java8.parse("public class Foo { String[] s = { \"Ven\\u00E4j\\u00E4\" }; }"); + } + + @Test + public void testUnicodeEscapes2() { + java.parse("\n" + + "public final class TimeZoneNames_zh_TW extends TimeZoneNamesBundle {\n" + + "\n" + + " String ACT[] = new String[] {\"Acre \\u6642\\u9593\", \"ACT\",\n" + + " \"Acre \\u590f\\u4ee4\\u6642\\u9593\", \"ACST\",\n" + + " \"Acre \\u6642\\u9593\", \"ACT\"};" + + "}"); + } + @Test public final void testGetFirstASTNameImageNull() { java4.parse(ABSTRACT_METHOD_LEVEL_CLASS_DECL); From fd375e4bcd8bea219385958a29b7dc8a014dd949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 16:11:35 +0200 Subject: [PATCH 014/136] More opts --- .../lang/ast/impl/javacc/EscapeTracker.java | 57 ++++++++++++------- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 4 ++ .../net/sourceforge/pmd/util/StringUtil.java | 16 ++++++ 3 files changed, 56 insertions(+), 21 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java index 4de40a65ee..1fc387c5d9 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java @@ -15,9 +15,14 @@ import java.io.EOFException; import net.sourceforge.pmd.util.document.Chars; /** - * Records where escapes occurred in the input document. This is quite - * an inefficient way to deal with it, yet in the common case where there - * are few/no escapes, it's enough I think. + * Records where escapes occurred in the input document. This is optimal + * for the case where there are few/no escapes. + * + *

This implementation can handle arbitrary length differences between + * the escape and its translation, provided the translation is always smaller + * than the escape. + * - C++ translates newline escapes (1 or 2 chars) to zero chars (an important corner case) + * - Java translates arbitrary-length unicode escapes (>= 6 chars) to 1 char */ class EscapeTracker { @@ -26,20 +31,20 @@ class EscapeTracker { static final EOFException EOF = new EOFException(); /* - * Offsets in the input buffer where a unicode escape occurred. - * Represented as tuples (off, len, invalid) where + * Escapes are encoded as tuples (off, after, invalid) where * - off is the offset in the source file where the escape occurred - * - len is the length of the escape in the input file, eg for \ u 00a0 will be 6 + * - after is the index of the char following the escape in the input file * - invalid is the last offset in the buffer which contains the translated chars (exclusive) * - * Eg for "a\u00a0b" (translates as "a b"), the buffer looks like - * [a u00a0b] - * ^ (off) this char has been replaced with the translated value of the escape - * ^^^^^ (off + len - invalid) these characters are only present in the input, we jump over them when reading - * ^ (invalid) - * ^ (off + len) + * Eg for "_\u00a0_" (translates as "_ _"), the buffer looks like + * [_ u00a0_] + * ^ (off) this char has been replaced with the translated value of the escape + * ^^^^^ (after - invalid) these characters are only present in the input, we jump over them when reading + * ^ (invalid) offset at which to jump to 'after' + * ^ (after) first char after the escape + * ^^^^^^ (after - off) total length of the escape in the input * - * The escape record is (1,6,2) + * The escape record is (1,7,2) * * When reading the buffer we'll copy two blocks * * "a " @@ -60,14 +65,14 @@ class EscapeTracker { */ void recordEscape(int offsetInInput, int lengthInInput, int lengthInOutput) { if (nextFreeIdx + 1 >= escapeRecords.length) { - // add 1 to not stay stuck at zero + // add 1 to not stay stuck at zero, needs to remain a multiple of RECORD_SIZE int[] newOffsets = new int[(escapeRecords.length + 1) * RECORD_SIZE]; System.arraycopy(escapeRecords, 0, newOffsets, 0, escapeRecords.length); this.escapeRecords = newOffsets; } escapeRecords[nextFreeIdx++] = offsetInInput; - escapeRecords[nextFreeIdx++] = lengthInInput; + escapeRecords[nextFreeIdx++] = offsetInInput + lengthInInput; escapeRecords[nextFreeIdx++] = offsetInInput + lengthInOutput; } @@ -76,18 +81,20 @@ class EscapeTracker { return escapeRecords[idx]; } - int inLen(int idx) { + int indexAfter(int idx) { assert idx < nextFreeIdx; return escapeRecords[idx + 1]; } + int invalidIdx(int idx) { assert idx < nextFreeIdx; return escapeRecords[idx + 2]; } - int indexAfter(int idx) { - return inOff(idx) + inLen(idx); + int inLen(int idx) { + assert idx < nextFreeIdx; + return indexAfter(idx) - inOff(idx); } /** @@ -193,6 +200,15 @@ class EscapeTracker { if (nextEscape <= 0) { pos -= numChars; // then there were no escapes before the 'pos' + } else if (numChars == 1) { + // fast path, very common + int esc = nextEscape - RECORD_SIZE; // >= 0 because of condition above + if (indexAfter(esc) == pos) { // jump back over the escape + pos = invalidIdx(esc) - 1; + nextEscape = esc; + } else { + pos--; + } } else { int newOff = pos; for (int i = nextEscape - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { @@ -247,12 +263,11 @@ class EscapeTracker { public Chars getMarkImage() { if (markEscape == nextEscape) { - // no escape in the marked range + // no escape in the marked range, this is the fast path return buf.slice(mark, markLength()); } StringBuilder sb = new StringBuilder(markLength()); - int prevLength = sb.length(); int cur = mark; int esc = markEscape; @@ -263,7 +278,7 @@ class EscapeTracker { } // no more escape in the range, append everything until the pos buf.appendChars(sb, cur, pos - cur); - assert sb.length() - prevLength == markLength() : sb + " should have length " + markLength(); + assert sb.length() == markLength() : sb + " should have length " + markLength(); return Chars.wrap(sb, true); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index 0c113a3a01..c3cf6052fc 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -141,6 +141,10 @@ public class JavaccToken implements GenericToken { return image; } + public CharSequence getImageCs() { + return image; + } + public boolean imageEquals(CharSequence charSequence) { return StringUtils.equals(image, charSequence); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java index 15c779f735..d1bd072f53 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java @@ -14,6 +14,7 @@ import org.apache.commons.lang3.StringUtils; import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.internal.util.AssertionUtil; +import net.sourceforge.pmd.util.document.Chars; /** * A number of String-specific utility methods for use by PMD or its IDE @@ -145,6 +146,21 @@ public final class StringUtil { return col; } + /** + * Like {@link StringBuilder#append(CharSequence)}, but uses an optimized + * implementation if the charsequence happens to be a {@link Chars}. {@link StringBuilder} + * already optimises the cases where the charseq is a string, a StringBuilder, + * or a stringBuffer. This is especially useful in parsers. + */ + public static StringBuilder append(StringBuilder sb, CharSequence charSeq) { + if (charSeq instanceof Chars) { + ((Chars) charSeq).appendChars(sb, 0, charSeq.length()); + return sb; + } else { + return sb.append(charSeq); + } + } + /** * Returns the substring following the last occurrence of the * given character. If the character doesn't occur, returns From 41d747ead2280800adbb2e5ef9696afd82ef5179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 16:27:53 +0200 Subject: [PATCH 015/136] Abstract backslash escape readers --- .../impl/javacc/BackslashEscapeReader.java | 68 +++++++++++++++++++ .../ast/impl/javacc/EscapeAwareReader.java | 37 ++++++++-- .../lang/ast/impl/javacc/EscapeTracker.java | 3 + .../lang/ast/impl/javacc/JavaInputReader.java | 49 +++---------- .../pmd/lang/cpp/ast/CppEscapeReader.java | 37 ++-------- 5 files changed, 117 insertions(+), 77 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java new file mode 100644 index 0000000000..a179452fd9 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java @@ -0,0 +1,68 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc; + +import static java.lang.Integer.min; + +import java.io.IOException; + +import net.sourceforge.pmd.util.document.Chars; + +/** + * A base class for readers that handle escapes starting with a backslash. + */ +public abstract class BackslashEscapeReader extends EscapeAwareReader { + + private static final char BACKSLASH = '\\'; + + /** + * An offset until which we read backslashes and decided they were not + * an escape. The read procedure may cut off in the middle of the escape, + * and turn an even num of backslashes into an odd one, so until we crossed + * this offset, backslashes are not treated specially. + */ + private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; + + + public BackslashEscapeReader(Chars input) { + super(input); + } + + @Override + protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { + int off = this.bufpos; + boolean noBackSlash = false; + int notEscapeEnd = this.savedNotEscapeSpecialEnd; + while (off < maxOff && (noBackSlash = input.charAt(off) != BACKSLASH || notEscapeEnd < off)) { + off++; + } + + if (noBackSlash || off == maxOff) { + this.bufpos = off; + return off; + } + + return handleBackslash(maxOff, off); + } + + protected abstract int handleBackslash(int maxOff, int firstBackslashOff) throws IOException; + + @Override + protected int recordEscape(int startOffsetInclusive, int lengthInSource, int translatedLength) { + this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; + return super.recordEscape(startOffsetInclusive, lengthInSource, translatedLength); + } + + protected int abortEscape(int off, int maxOff) { + // not an escape sequence + int min = min(maxOff, off); + // save the number of backslashes that are part of the escape, + // might have been cut in half by the maxReadahead + this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; + this.bufpos = min; + return min; + } + +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java index 8b0f8eb0e1..2e813d5fc3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java @@ -10,16 +10,27 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import static java.lang.Integer.min; +import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; +import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; /** - * A reader that can interpret escapes in its input text. It records where - * escapes occurred, and can translate an offset in the translated - * input document to a line+column position in the original input. + * A reader that may interpret escapes in its input text. It records + * where escapes occurred, and can translate an offset in the translated + * document (the "output") to a line/column/offset coordinates in the + * original input. It uses a single char buffer to store both input and + * translated output, and is overall very optimised for the case where + * there are very few escapes. {@link CharStream} is the API to navigate + * on a translated document (with arbitrary backtrack abilities). + * + *

This is useful to back a {@link CharStream} for JavaCC implementation, + * but can also be used as a plain {@link Reader} if using other parser/lexer + * implementations. The reader behaviour is optimised for block IO and has + * poor char-by-char performance. Use a {@link BufferedReader} if you need it. * *

The default implementation does not perform any escape translation. */ @@ -39,7 +50,7 @@ public class EscapeAwareReader extends Reader { final EscapeTracker escapes = new EscapeTracker(); public EscapeAwareReader(Chars input) { - assert input != null; + AssertionUtil.requireParamNotNull("input", input); this.input = input.mutableCopy(); bufpos = 0; } @@ -91,15 +102,17 @@ public class EscapeAwareReader extends Reader { /** * Returns the max offset, EXclusive, with which we can cut the input - * array from the bufpos to dump it into the output array. This sets - * the bufpos to where we should start the next jump. + * array from the bufpos to dump it into the output array. This must + * set the {@link #bufpos} to where we should start reading next (INclusive). + * If applicable, it must also replace in the buffer the start of + * the escape with its translation. */ protected int gobbleMaxWithoutEscape(int maxOff) throws IOException { return this.bufpos = maxOff; } protected int recordEscape(final int startOffsetInclusive, int lengthInSource, int translatedLength) { - assert lengthInSource > 0 && startOffsetInclusive >= 0; + assert lengthInSource > 0 && lengthInSource >= translatedLength && startOffsetInclusive >= 0; this.escapes.recordEscape(startOffsetInclusive, lengthInSource, translatedLength); this.bufpos = startOffsetInclusive + lengthInSource; return startOffsetInclusive + translatedLength; @@ -142,10 +155,20 @@ public class EscapeAwareReader extends Reader { return escapes.inputOffsetAt(outputOffset); } + /** + * The parameter is an *input* offset, if you got this offset from + * somewhere else than the input buffer you must first translate it + * back with {@link #inputOffset(int)}. This implementation is very + * inefficient but currently is only used for error messages (which + * obviously are exceptional). + */ public int getLine(int idxInInput) { return StringUtil.lineNumberAt(input, idxInInput); } + /** + * @see #getLine(int) + */ public int getColumn(int idxInInput) { return StringUtil.columnNumberAt(input, idxInInput); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java index 1fc387c5d9..ec55a226b8 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java @@ -23,6 +23,9 @@ import net.sourceforge.pmd.util.document.Chars; * than the escape. * - C++ translates newline escapes (1 or 2 chars) to zero chars (an important corner case) * - Java translates arbitrary-length unicode escapes (>= 6 chars) to 1 char + * + *

This class is tightly coupled to what {@link EscapeAwareReader} + * does with its buffer. */ class EscapeTracker { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java index 68f1970809..3c1915c7d3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java @@ -8,53 +8,29 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import static java.lang.Integer.min; - -import java.io.BufferedReader; import java.io.IOException; import net.sourceforge.pmd.util.document.Chars; /** - * An implementation of java.io.Reader that translates Java unicode escapes. - * This implementation has efficient block IO but poor char-by-char performance. - * If this is required, wrap it into a {@link BufferedReader}. + * An implementation of {@link EscapeAwareReader} that translates Java + * unicode escapes. */ @SuppressWarnings("PMD.AssignmentInOperand") -public final class JavaInputReader extends EscapeAwareReader { - - /** - * An offset until which we read backslashes and decided they were not - * an escape. The read procedure may cut off in the middle of the escape, - * and turn an even num of backslashes into an odd one, so until we crossed - * this offset, backslashes are not treated specially. - */ - private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; +public final class JavaInputReader extends BackslashEscapeReader { public JavaInputReader(Chars input) { super(input); } @Override - protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { - int off = this.bufpos; - boolean noBackSlash = false; - int notEscapeEnd = this.savedNotEscapeSpecialEnd; - while (off < maxOff && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { - off++; - } - - if (noBackSlash) { - this.bufpos = off; - return off; - } - - final int firstBslashOff = off; + protected int handleBackslash(final int maxOff, final int firstBackslashOff) throws IOException { + int off = firstBackslashOff; while (off < input.length() && input.charAt(off) == '\\') { off++; } - int bslashCount = off - firstBslashOff; + int bslashCount = off - firstBackslashOff; // is there an escape at offset firstBslashOff? if ((bslashCount & 1) == 1 // odd number of backslashes && off < input.length() && input.charAt(off) == 'u') { // at least one 'u' @@ -63,17 +39,10 @@ public final class JavaInputReader extends EscapeAwareReader { // consume all the 'u's off++; } - int end = replaceFirstBackslashWithEscape(firstBslashOff, off - 1); - this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - return recordEscape(firstBslashOff, end - firstBslashOff, 1); + int end = replaceFirstBackslashWithEscape(firstBackslashOff, off - 1); + return recordEscape(firstBackslashOff, end - firstBackslashOff, 1); } else { - // not an escape sequence - int min = min(maxOff, off); - // save the number of backslashes that are part of the escape, - // might have been cut in half by the maxReadahead - this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; - this.bufpos = min; - return min; + return abortEscape(off, maxOff); } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java index 99f4581f5e..d587ce86f7 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -4,53 +4,30 @@ package net.sourceforge.pmd.lang.cpp.ast; -import static java.lang.Integer.min; - -import java.io.IOException; - -import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.BackslashEscapeReader; import net.sourceforge.pmd.util.document.Chars; -public class CppEscapeReader extends EscapeAwareReader { +public class CppEscapeReader extends BackslashEscapeReader { private static final char NEWLINE = '\n'; private static final char CARRIAGE_RETURN = '\r'; - private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - public CppEscapeReader(Chars input) { super(input); } @Override - protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { - int off = this.bufpos; - boolean noBackSlash = false; - int notEscapeEnd = this.savedNotEscapeSpecialEnd; - while (off < maxOff && (noBackSlash = input.charAt(off) != '\\' || notEscapeEnd < off)) { - off++; - } + protected int handleBackslash(int maxOff, final int backSlashOff) { + int off = backSlashOff; - if (noBackSlash || off == maxOff) { - this.bufpos = off; - return off; - } - - final int backSlackOff = off++; if (input.charAt(off) == NEWLINE) { - return recordEscape(backSlackOff, 2, 0); + return recordEscape(backSlashOff, 2, 0); } else if (input.charAt(off) == CARRIAGE_RETURN) { if (input.charAt(++off) == NEWLINE) { - return recordEscape(backSlackOff, 3, 0); + return recordEscape(backSlashOff, 3, 0); } } - // not an escape sequence - int min = min(maxOff, off); - // save the number of backslashes that are part of the escape, - // might have been cut in half by the maxReadahead - this.savedNotEscapeSpecialEnd = min < off ? off : Integer.MAX_VALUE; - this.bufpos = min; - return min; + return abortEscape(off, maxOff); } } From dc8ac96013fb1e8691ef3ffcd88dfe20dbb4fd22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 16:48:54 +0200 Subject: [PATCH 016/136] Fix cherry-pick --- .../pmd/lang/ast/impl/javacc/JavaccTokenDocument.java | 7 ++++++- .../lang/ast/impl/{io => javacc}/JavaInputReaderTest.java | 4 +--- 2 files changed, 7 insertions(+), 4 deletions(-) rename pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/{io => javacc}/JavaInputReaderTest.java (97%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index acf616d2ad..cc9776f7a0 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -23,7 +23,12 @@ public class JavaccTokenDocument extends TokenDocument { super(textDocument); } - + /** + * Create new (possibly) escaping reader for the given text. The default + * implementation doesn't do any escaping. + * + * @param text Source doc + */ public EscapeAwareReader newReader(Chars text) { return new EscapeAwareReader(text); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReaderTest.java similarity index 97% rename from pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReaderTest.java index 61c2de1725..7086c6f1cd 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/io/JavaInputReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReaderTest.java @@ -1,8 +1,7 @@ /* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ - -package net.sourceforge.pmd.lang.ast.impl.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.IOException; @@ -10,7 +9,6 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; import net.sourceforge.pmd.util.document.Chars; From 6142dc39785dfaf44fd71b0b9ee3a9a9027b1d0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 16:56:50 +0200 Subject: [PATCH 017/136] Rename java reader --- ...InputReader.java => JavaEscapeReader.java} | 4 ++-- .../ast/impl/javacc/CharStreamImplTest.java | 2 +- ...derTest.java => JavaEscapeReaderTest.java} | 20 +++++++++---------- .../pmd/lang/java/ast/JavaTokenDocument.java | 4 ++-- .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 4 ++-- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 4 ++-- 6 files changed, 19 insertions(+), 19 deletions(-) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{JavaInputReader.java => JavaEscapeReader.java} (96%) rename pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/{JavaInputReaderTest.java => JavaEscapeReaderTest.java} (88%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReader.java similarity index 96% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReader.java index 3c1915c7d3..57101082b3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReader.java @@ -17,9 +17,9 @@ import net.sourceforge.pmd.util.document.Chars; * unicode escapes. */ @SuppressWarnings("PMD.AssignmentInOperand") -public final class JavaInputReader extends BackslashEscapeReader { +public final class JavaEscapeReader extends BackslashEscapeReader { - public JavaInputReader(Chars input) { + public JavaEscapeReader(Chars input) { super(input); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java index 2d57cae381..b2cd262fd8 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java @@ -142,7 +142,7 @@ public class CharStreamImplTest { return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)) { @Override public EscapeAwareReader newReader(Chars text) { - return new JavaInputReader(text); + return new JavaEscapeReader(text); } }); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java similarity index 88% rename from pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReaderTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java index 7086c6f1cd..cac67177c8 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaInputReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java @@ -12,11 +12,11 @@ import org.junit.Test; import net.sourceforge.pmd.util.document.Chars; -public class JavaInputReaderTest { +public class JavaEscapeReaderTest { @NonNull - public JavaInputReader readString(String input) { - return new JavaInputReader(Chars.wrap(input, true)); + public JavaEscapeReader readString(String input) { + return new JavaEscapeReader(Chars.wrap(input, true)); } @@ -24,7 +24,7 @@ public class JavaInputReaderTest { public void testSimpleRead() throws IOException { String input = "abcdede"; - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[12]; @@ -39,7 +39,7 @@ public class JavaInputReaderTest { public void testNotAnEscape1Read() throws IOException { String input = "abc\\dede"; - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[12]; @@ -54,7 +54,7 @@ public class JavaInputReaderTest { public void testNotAnEscape1Read2() throws IOException { String input = "abc\\\\\\dede"; - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[12]; @@ -70,7 +70,7 @@ public class JavaInputReaderTest { String input = "abc\\\\\\dede"; // ^ - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[12]; @@ -95,7 +95,7 @@ public class JavaInputReaderTest { public void testAnEscapeStopAtEnd() throws IOException { String input = "abc\\\\\\u00a0dede"; - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[12]; @@ -115,7 +115,7 @@ public class JavaInputReaderTest { public void testSeveralEscapes() throws IOException { String input = "abc\\\\\\u00a0d\\uu00a0ede"; - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[20]; @@ -135,7 +135,7 @@ public class JavaInputReaderTest { public void testAnEscapeInsideBlock() throws IOException { String input = "abc\\\\\\u00a0dede\\u00a0"; - try (JavaInputReader r = readString(input)) { + try (JavaEscapeReader r = readString(input)) { char[] chars = new char[12]; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index a1d74fe154..92416a814f 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -16,7 +16,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; @@ -48,7 +48,7 @@ final class JavaTokenDocument extends JavaccTokenDocument { @Override public EscapeAwareReader newReader(Chars text) { - return new JavaInputReader(text); + return new JavaEscapeReader(text); } @Override diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index 48a62f5131..1465435a22 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -8,7 +8,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; @@ -27,7 +27,7 @@ public class JSPTokenizer extends JavaCCTokenizer { return new JavaccTokenDocument(textDoc) { @Override public EscapeAwareReader newReader(Chars text) { - return new JavaInputReader(text); + return new JavaEscapeReader(text); } }; } diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index ba239ef3cf..312481b653 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -8,7 +8,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaInputReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; @@ -30,7 +30,7 @@ public class VfTokenizer extends JavaCCTokenizer { return new JavaccTokenDocument(textDoc) { @Override public EscapeAwareReader newReader(Chars text) { - return new JavaInputReader(text); + return new JavaEscapeReader(text); } }; } From e93e5b0a4c0c8939b739d20669377f6d5f688f4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 17:03:09 +0200 Subject: [PATCH 018/136] Move into .io package --- javacc-wrapper.xml | 4 ++-- .../net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java | 2 +- .../sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java | 1 + .../pmd/lang/ast/impl/javacc/JavaccTokenDocument.java | 2 ++ .../pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java | 1 + .../ast/impl/javacc/{ => io}/BackslashEscapeReader.java | 2 +- .../pmd/lang/ast/impl/javacc/{ => io}/CharStream.java | 3 ++- .../lang/ast/impl/javacc/{ => io}/EscapeAwareReader.java | 6 +----- .../pmd/lang/ast/impl/javacc/{ => io}/EscapeTracker.java | 8 ++------ .../lang/ast/impl/javacc/{ => io}/JavaEscapeReader.java | 6 +----- .../pmd/lang/ast/impl/javacc/CharStreamImplTest.java | 3 ++- .../pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java | 1 + pmd-cpp/etc/grammar/Cpp.jj | 2 +- .../main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java | 4 ++-- .../net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java | 2 +- .../java/net/sourceforge/pmd/cpd/CppCharStreamTest.java | 2 +- pmd-java/etc/grammar/Java.jjt | 2 +- .../main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java | 2 +- .../net/sourceforge/pmd/lang/java/ast/JavaParser.java | 2 +- .../sourceforge/pmd/lang/java/ast/JavaTokenDocument.java | 6 +++--- .../net/sourceforge/pmd/lang/java/ast/TestExtensions.kt | 2 +- pmd-javascript/etc/grammar/Ecmascript5.jj | 2 +- .../java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java | 2 +- pmd-jsp/etc/grammar/Jsp.jjt | 2 +- .../main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java | 6 +++--- .../java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java | 2 +- pmd-matlab/etc/grammar/Matlab.jj | 2 +- .../java/net/sourceforge/pmd/cpd/MatlabTokenizer.java | 2 +- pmd-modelica/etc/grammar/Modelica.jjt | 2 +- .../java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java | 2 +- .../sourceforge/pmd/lang/modelica/ast/ModelicaParser.java | 2 +- pmd-objectivec/etc/grammar/ObjectiveC.jj | 2 +- .../java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java | 2 +- pmd-plsql/etc/grammar/PLSQL.jjt | 2 +- .../main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java | 2 +- .../net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java | 2 +- pmd-python/etc/grammar/Python.jj | 2 +- .../java/net/sourceforge/pmd/cpd/PythonTokenizer.java | 2 +- pmd-visualforce/etc/grammar/Vf.jjt | 2 +- .../main/java/net/sourceforge/pmd/cpd/VfTokenizer.java | 6 +++--- .../java/net/sourceforge/pmd/lang/vf/ast/VfParser.java | 2 +- pmd-vm/etc/grammar/Vm.jjt | 2 +- .../java/net/sourceforge/pmd/lang/vm/ast/VmParser.java | 2 +- 43 files changed, 55 insertions(+), 60 deletions(-) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/BackslashEscapeReader.java (97%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/CharStream.java (97%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/EscapeAwareReader.java (97%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/EscapeTracker.java (98%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/JavaEscapeReader.java (95%) diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index b9cf9b1ba1..6addedec11 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -60,7 +60,7 @@ - + @@ -407,7 +407,7 @@ public final class ${token-constants-name} \{${line.separator} * be used as a basis for a CPD Tokenizer. */ @net.sourceforge.pmd.annotation.InternalApi - public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.CharStream cs) { + public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.io.CharStream cs) { return new %%%TOKEN_MGR_NAME%%%(cs); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 7f6b906172..d39486f6f4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -14,9 +14,9 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; import net.sourceforge.pmd.util.document.io.TextFile; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index c3cf6052fc..c8d28840c6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import org.apache.commons.lang3.StringUtils; import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextRegion; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index cc9776f7a0..9529abf521 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -8,6 +8,8 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index c37da63d78..f2a0d0b90c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -10,6 +10,7 @@ import net.sourceforge.pmd.lang.Parser; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.RootNode; import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java similarity index 97% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java index a179452fd9..d43677c604 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc; +package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java similarity index 97% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index 3a14835da1..13908ca02f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -2,12 +2,13 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc; +package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.EOFException; import java.io.IOException; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java similarity index 97% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index 2e813d5fc3..c26641a58d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -2,11 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; +package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java similarity index 98% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java index ec55a226b8..d63d824a17 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTracker.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java @@ -2,11 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; +package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.max; @@ -134,7 +130,7 @@ class EscapeTracker { return res.append('}').toString(); } - /** Backend for a CharStream. */ + /** Backend for a CharStream. Maintains a current position and a mark. */ class Cursor { /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java similarity index 95% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index 57101082b3..db9e49d203 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -2,11 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; +package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.IOException; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java index b2cd262fd8..fe4362ca00 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java @@ -4,7 +4,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import java.io.EOFException; @@ -14,6 +13,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java index cac67177c8..84a109591a 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java @@ -9,6 +9,7 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.util.document.Chars; diff --git a/pmd-cpp/etc/grammar/Cpp.jj b/pmd-cpp/etc/grammar/Cpp.jj index eed7cc58cf..f292dd019c 100644 --- a/pmd-cpp/etc/grammar/Cpp.jj +++ b/pmd-cpp/etc/grammar/Cpp.jj @@ -32,7 +32,7 @@ options { PARSER_BEGIN(CppParserImpl) package net.sourceforge.pmd.lang.cpp.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public final class CppParserImpl { diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 9d0bcb4a93..7a15244762 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -13,10 +13,10 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; import net.sourceforge.pmd.util.document.Chars; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java index d587ce86f7..e0531eca21 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -4,7 +4,7 @@ package net.sourceforge.pmd.lang.cpp.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.BackslashEscapeReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeReader; import net.sourceforge.pmd.util.document.Chars; public class CppEscapeReader extends BackslashEscapeReader { diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 55c4aef643..1942bbc2a9 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -11,7 +11,7 @@ import java.io.IOException; import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; public class CppCharStreamTest { diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index b2eb1fbba4..ab9fe30349 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -240,7 +240,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Set; import java.util.Map; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.Node; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index aa304b6ab2..c58e86b136 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -13,9 +13,9 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index ea8a1b2b1b..2f77e0bfb0 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -4,10 +4,10 @@ package net.sourceforge.pmd.lang.java.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 92416a814f..01e6e1bda4 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -14,11 +14,11 @@ import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.WHITESPACE; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/TestExtensions.kt b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/TestExtensions.kt index efb23d035e..6f77946eb9 100644 --- a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/TestExtensions.kt +++ b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/TestExtensions.kt @@ -8,9 +8,9 @@ import com.github.oowekyala.treeutils.matchers.TreeNodeWrapper import io.kotest.matchers.Matcher import io.kotest.matchers.MatcherResult import io.kotest.matchers.collections.shouldBeEmpty -import io.kotest.matchers.types.shouldBeInstanceOf import io.kotest.matchers.shouldBe import io.kotest.matchers.shouldNotBe +import io.kotest.matchers.types.shouldBeInstanceOf import net.sourceforge.pmd.internal.util.IteratorUtil import net.sourceforge.pmd.lang.ast.Node import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken diff --git a/pmd-javascript/etc/grammar/Ecmascript5.jj b/pmd-javascript/etc/grammar/Ecmascript5.jj index a9cf14bcbb..a1b77891ce 100644 --- a/pmd-javascript/etc/grammar/Ecmascript5.jj +++ b/pmd-javascript/etc/grammar/Ecmascript5.jj @@ -15,7 +15,7 @@ options { PARSER_BEGIN(Ecmascript5ParserImpl) package net.sourceforge.pmd.lang.ecmascript5.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class Ecmascript5ParserImpl { diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index d66d74949f..959d446fe6 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5TokenKinds; diff --git a/pmd-jsp/etc/grammar/Jsp.jjt b/pmd-jsp/etc/grammar/Jsp.jjt index 9e4483a441..ca66ff9571 100644 --- a/pmd-jsp/etc/grammar/Jsp.jjt +++ b/pmd-jsp/etc/grammar/Jsp.jjt @@ -30,7 +30,7 @@ options { PARSER_BEGIN(JspParserImpl) package net.sourceforge.pmd.lang.jsp.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index 1465435a22..c5ac10410e 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -6,11 +6,11 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java index 6d8cf7f3a0..fe8a329dcc 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java @@ -6,10 +6,10 @@ package net.sourceforge.pmd.lang.jsp.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; /** diff --git a/pmd-matlab/etc/grammar/Matlab.jj b/pmd-matlab/etc/grammar/Matlab.jj index 3ff1c8b27c..6220de48c8 100644 --- a/pmd-matlab/etc/grammar/Matlab.jj +++ b/pmd-matlab/etc/grammar/Matlab.jj @@ -21,7 +21,7 @@ options { PARSER_BEGIN(MatlabParserImpl) package net.sourceforge.pmd.lang.matlab.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class MatlabParserImpl { diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java index 9459c44696..17b0ba9003 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.matlab.ast.MatlabTokenKinds; diff --git a/pmd-modelica/etc/grammar/Modelica.jjt b/pmd-modelica/etc/grammar/Modelica.jjt index e2e7d63121..9d2cdfa526 100644 --- a/pmd-modelica/etc/grammar/Modelica.jjt +++ b/pmd-modelica/etc/grammar/Modelica.jjt @@ -49,7 +49,7 @@ options { PARSER_BEGIN(ModelicaParserImpl) package net.sourceforge.pmd.lang.modelica.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; class ModelicaParserImpl { diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java index 3258a3cda7..eabfc401d8 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.modelica.ast.ModelicaTokenKinds; diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java index 058941d176..3cf646d962 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java @@ -4,7 +4,7 @@ package net.sourceforge.pmd.lang.modelica.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-objectivec/etc/grammar/ObjectiveC.jj b/pmd-objectivec/etc/grammar/ObjectiveC.jj index 171ca581fa..c4679634a1 100644 --- a/pmd-objectivec/etc/grammar/ObjectiveC.jj +++ b/pmd-objectivec/etc/grammar/ObjectiveC.jj @@ -21,7 +21,7 @@ package net.sourceforge.pmd.lang.objectivec.ast; import java.io.*; import java.util.*; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; /** diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java index acccfcd24a..1e4bfb3e5a 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.objectivec.ast.ObjectiveCTokenKinds; diff --git a/pmd-plsql/etc/grammar/PLSQL.jjt b/pmd-plsql/etc/grammar/PLSQL.jjt index 090f90e235..a44efb486a 100644 --- a/pmd-plsql/etc/grammar/PLSQL.jjt +++ b/pmd-plsql/etc/grammar/PLSQL.jjt @@ -154,7 +154,7 @@ package net.sourceforge.pmd.lang.plsql.ast; import java.io.*; import net.sourceforge.pmd.lang.ast.Node; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class PLSQLParserImpl { diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index 77abbf8794..72a90a80b2 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -8,8 +8,8 @@ import java.util.Properties; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.plsql.ast.PLSQLTokenKinds; public class PLSQLTokenizer extends JavaCCTokenizer { diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java index 5cf6fa5ef7..ab7e455512 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java @@ -6,10 +6,10 @@ package net.sourceforge.pmd.lang.plsql.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; public class PLSQLParser extends JjtreeParserAdapter { diff --git a/pmd-python/etc/grammar/Python.jj b/pmd-python/etc/grammar/Python.jj index 8087042e7c..cba9a4fce6 100644 --- a/pmd-python/etc/grammar/Python.jj +++ b/pmd-python/etc/grammar/Python.jj @@ -17,7 +17,7 @@ PARSER_BEGIN(PythonParserImpl) package net.sourceforge.pmd.lang.python.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class PythonParserImpl { diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 70755980ea..de6d1b2b01 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -10,7 +10,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds; diff --git a/pmd-visualforce/etc/grammar/Vf.jjt b/pmd-visualforce/etc/grammar/Vf.jjt index 2af403e0cb..b123075211 100644 --- a/pmd-visualforce/etc/grammar/Vf.jjt +++ b/pmd-visualforce/etc/grammar/Vf.jjt @@ -13,7 +13,7 @@ options { PARSER_BEGIN(VfParserImpl) package net.sourceforge.pmd.lang.vf.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class VfParserImpl { diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 312481b653..80d768ff7a 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -6,9 +6,9 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java index b740cc8c63..f27ad84dd0 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.vf.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-vm/etc/grammar/Vm.jjt b/pmd-vm/etc/grammar/Vm.jjt index 678ed8de31..5dc623deb7 100644 --- a/pmd-vm/etc/grammar/Vm.jjt +++ b/pmd-vm/etc/grammar/Vm.jjt @@ -46,7 +46,7 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java index 845653d06c..e555891db1 100644 --- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java +++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java @@ -6,11 +6,11 @@ package net.sourceforge.pmd.lang.vm.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; /** From ea62ad7f2ed8efa260409821040594fd29bd9f27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 17:08:34 +0200 Subject: [PATCH 019/136] Remove lazy image token Slicing Chars will be enough --- .../pmd/lang/java/ast/JavaTokenDocument.java | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 01e6e1bda4..1c62d083c0 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -10,7 +10,6 @@ import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.MULTI_LINE_COMMEN import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.RSIGNEDSHIFT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.RUNSIGNEDSHIFT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.SINGLE_LINE_COMMENT; -import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.WHITESPACE; import org.checkerframework.checker.nullness.qual.Nullable; @@ -70,19 +69,6 @@ final class JavaTokenDocument extends JavaccTokenDocument { jcs.getEndOffset(), jcs.getTokenDocument() ); - - case WHITESPACE: - // We don't create a new string for the image of whitespace tokens eagerly - - // It's unlikely that anybody cares about that, and since - // they're still 30% of all tokens this is advantageous - return new LazyImageToken( - kind, - jcs.getStartOffset(), - jcs.getEndOffset(), - jcs.getTokenDocument() - ); - default: return super.createToken(kind, jcs, image); } @@ -92,18 +78,6 @@ final class JavaTokenDocument extends JavaccTokenDocument { return token instanceof GTToken ? ((GTToken) token).realKind : token.kind; } - private static final class LazyImageToken extends JavaccToken { - - LazyImageToken(int kind, int startInclusive, int endExclusive, JavaccTokenDocument document) { - super(kind, null, startInclusive, endExclusive, document); - } - - @Override - public String getImage() { - return document.getTextDocument().sliceText(getRegion()).toString(); - } - } - private static final class GTToken extends JavaccToken { final int realKind; @@ -114,6 +88,4 @@ final class JavaTokenDocument extends JavaccTokenDocument { } } - - } From 5d7e57ca9b059915eb79beaca9a36b3b4878da51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 17:14:19 +0200 Subject: [PATCH 020/136] Move tests --- .../pmd/lang/ast/GenericToken.java | 22 +++++++++++++++++-- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 4 ---- .../javacc/{ => io}/CharStreamImplTest.java | 6 ++--- .../javacc/{ => io}/JavaEscapeReaderTest.java | 4 ++-- 4 files changed, 25 insertions(+), 11 deletions(-) rename pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/CharStreamImplTest.java (96%) rename pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/{ => io}/JavaEscapeReaderTest.java (97%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java index 40f58e84f2..f4886fcfa6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java @@ -6,6 +6,8 @@ package net.sourceforge.pmd.lang.ast; import java.util.Iterator; +import org.apache.commons.lang3.StringUtils; + import net.sourceforge.pmd.internal.util.IteratorUtil; import net.sourceforge.pmd.util.document.Reportable; import net.sourceforge.pmd.util.document.TextRegion; @@ -31,18 +33,33 @@ public interface GenericToken> extends Comparable, T getPreviousComment(); /** - * Returns the token's text. + * Returns the token's text as a string. */ default String getImage() { return getImageCs().toString(); } + /** - * Returns the image as a {@link CharSequence}. + * Returns the text of the token as a char sequence. + * This should be preferred when you can use eg {@link StringUtils} + * to do some processing, without having to create a string. */ CharSequence getImageCs(); + /** + * Returns true if the image of this token equals + * the given charsequence. This does not create a + * string. + * + * @param charSeq A character sequence + */ + default boolean imageEquals(CharSequence charSeq) { + return StringUtils.equals(getImageCs(), charSeq); + } + + /** Returns a text region with the coordinates of this token. */ TextRegion getRegion(); @@ -52,6 +69,7 @@ public interface GenericToken> extends Comparable, */ boolean isEof(); + /** * Returns true if this token is implicit, ie was inserted artificially * and has a zero-length image. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index c8d28840c6..cd4f087df9 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -142,10 +142,6 @@ public class JavaccToken implements GenericToken { return image; } - public CharSequence getImageCs() { - return image; - } - public boolean imageEquals(CharSequence charSequence) { return StringUtils.equals(image, charSequence); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java similarity index 96% rename from pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index fe4362ca00..660df96ca5 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -2,8 +2,9 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc; +package net.sourceforge.pmd.lang.ast.impl.javacc.io; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import java.io.EOFException; @@ -13,8 +14,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java similarity index 97% rename from pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index 84a109591a..492afc7f06 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -1,7 +1,8 @@ /* * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc; + +package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.IOException; @@ -9,7 +10,6 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.util.document.Chars; From ad9d83a29478202167ec29c3e54b9d566d2915a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 18 Apr 2020 17:22:34 +0200 Subject: [PATCH 021/136] Cleanup cursor --- .../pmd/lang/ast/impl/javacc/io/CharStream.java | 3 +-- .../pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java | 8 +++++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index 13908ca02f..af2028818e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -31,8 +31,7 @@ public final class CharStream { */ public static CharStream create(JavaccTokenDocument doc) throws IOException { try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { - reader.translate(); - return new CharStream(doc, reader.escapes.new Cursor(reader.input)); + return new CharStream(doc, reader.translate()); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index c26641a58d..196dc6db49 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -52,10 +52,12 @@ public class EscapeAwareReader extends Reader { } /** - * Translate all the characters in the buffer. + * Translate all the input (in-place) in the buffer. This is fed to a + * cursor initialized to zero. */ - public int translate() throws IOException { - return readUnchecked(null, 0, Integer.MAX_VALUE); + EscapeTracker.Cursor translate() throws IOException { + readUnchecked(null, 0, Integer.MAX_VALUE); + return escapes.new Cursor(input); } From f6b8bdc975170014ab1e55515ee3d7f009044399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 19 Apr 2020 18:59:07 +0200 Subject: [PATCH 022/136] Replace old impl with newer one This uses "fragmented docs". This is less efficient when there are a lot of escapes, but: - it doesn't require the initial buffer copy - it doesn't mutate any buffer, so Chars can be readonly all the time - it handles escapes of arbitrary inLen/outLen ratio and is thus future proof - it is much more maintainable On the downside there is - efficiency (this data structure is "pointy") - clarity of the Reader implementation (one more state to support, ie reading not from the input buffer but from an escape) --- .../impl/javacc/io/BackslashEscapeReader.java | 4 +- .../lang/ast/impl/javacc/io/CharStream.java | 8 +- .../ast/impl/javacc/io/EscapeAwareReader.java | 48 ++- .../ast/impl/javacc/io/EscapeTracker.java | 305 ------------------ .../impl/javacc/io/FragmentedDocBuilder.java | 73 +++++ .../impl/javacc/io/FragmentedDocCursor.java | 172 ++++++++++ .../ast/impl/javacc/io/JavaEscapeReader.java | 11 +- .../impl/javacc/io/CharStreamImplTest.java | 2 - .../pmd/lang/cpp/ast/CppEscapeReader.java | 4 +- 9 files changed, 296 insertions(+), 331 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java index d43677c604..54724841a7 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java @@ -50,9 +50,9 @@ public abstract class BackslashEscapeReader extends EscapeAwareReader { protected abstract int handleBackslash(int maxOff, int firstBackslashOff) throws IOException; @Override - protected int recordEscape(int startOffsetInclusive, int lengthInSource, int translatedLength) { + protected int recordEscape(int startOffsetInclusive, int endOffsetExclusive, Chars translation) { this.savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - return super.recordEscape(startOffsetInclusive, lengthInSource, translatedLength); + return super.recordEscape(startOffsetInclusive, endOffsetExclusive, translation); } protected int abortEscape(int off, int maxOff) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index af2028818e..c52699936a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -19,9 +19,9 @@ import net.sourceforge.pmd.util.document.TextDocument; public final class CharStream { private final JavaccTokenDocument tokenDoc; - private final EscapeTracker.Cursor cursor; + private final FragmentedDocCursor cursor; - private CharStream(JavaccTokenDocument tokenDoc, EscapeTracker.Cursor cursor) { + private CharStream(JavaccTokenDocument tokenDoc, FragmentedDocCursor cursor) { this.tokenDoc = tokenDoc; this.cursor = cursor; } @@ -133,13 +133,13 @@ public final class CharStream { /** Returns the start offset of the current token (in the original source), inclusive. */ public int getStartOffset() { - return cursor.markOutOffset(); + return cursor.markInOffset(); } /** Returns the end offset of the current token (in the original source), exclusive. */ public int getEndOffset() { - return cursor.curOutOffset(); + return cursor.curInOffset(); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index 196dc6db49..9018931592 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -10,6 +10,8 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.Reader; +import org.checkerframework.checker.nullness.qual.Nullable; + import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; @@ -43,21 +45,25 @@ public class EscapeAwareReader extends Reader { /** Position of the next char to read in the input. */ protected int bufpos; /** Keep track of adjustments to make to the offsets, caused by unicode escapes. */ - final EscapeTracker escapes = new EscapeTracker(); + final FragmentedDocBuilder escapes; + + private Chars curEscape; + private int offInEscape; public EscapeAwareReader(Chars input) { AssertionUtil.requireParamNotNull("input", input); - this.input = input.mutableCopy(); + this.input = input; bufpos = 0; + escapes = new FragmentedDocBuilder(input); } /** * Translate all the input (in-place) in the buffer. This is fed to a * cursor initialized to zero. */ - EscapeTracker.Cursor translate() throws IOException { + FragmentedDocCursor translate() throws IOException { readUnchecked(null, 0, Integer.MAX_VALUE); - return escapes.new Cursor(input); + return escapes.newCursor(); } @@ -69,7 +75,8 @@ public class EscapeAwareReader extends Reader { return readUnchecked(cbuf, off, len); } - private int readUnchecked(char[] cbuf, int off, int len) throws IOException { + // if cbuf is null we just want to record escapes + private int readUnchecked(char @Nullable [] cbuf, int off, int len) throws IOException { ensureOpen(); if (this.bufpos == input.length()) { return -1; @@ -78,7 +85,24 @@ public class EscapeAwareReader extends Reader { len = min(len, input.length()); // remove Integer.MAX_VALUE int readChars = 0; - while (readChars < len && this.bufpos < input.length()) { + while (readChars < len && (this.bufpos < input.length() || curEscape != null)) { + if (curEscape != null) { + int toRead = min(len - readChars, curEscape.length() - offInEscape); + + if (cbuf != null) { + curEscape.getChars(0, cbuf, off + readChars, toRead); + } + readChars += toRead; + offInEscape += toRead; + + if (curEscape.length() == offInEscape) { + curEscape = null; + continue; + } else { + break; // len cut us off, we'll retry next time + } + } + int bpos = this.bufpos; int nextJump = gobbleMaxWithoutEscape(min(input.length(), bpos + len - readChars)); int newlyReadChars = nextJump - bpos; @@ -109,11 +133,13 @@ public class EscapeAwareReader extends Reader { return this.bufpos = maxOff; } - protected int recordEscape(final int startOffsetInclusive, int lengthInSource, int translatedLength) { - assert lengthInSource > 0 && lengthInSource >= translatedLength && startOffsetInclusive >= 0; - this.escapes.recordEscape(startOffsetInclusive, lengthInSource, translatedLength); - this.bufpos = startOffsetInclusive + lengthInSource; - return startOffsetInclusive + translatedLength; + protected int recordEscape(final int startOffsetInclusive, int endOffsetExclusive, Chars translation) { + assert endOffsetExclusive > startOffsetInclusive && startOffsetInclusive >= 0; + this.escapes.recordDelta(startOffsetInclusive, endOffsetExclusive, translation); + this.bufpos = endOffsetExclusive; + this.curEscape = translation; + this.offInEscape = 0; + return startOffsetInclusive; } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java deleted file mode 100644 index d63d824a17..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTracker.java +++ /dev/null @@ -1,305 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc.io; - -import static java.lang.Integer.max; - -import java.io.EOFException; - -import net.sourceforge.pmd.util.document.Chars; - -/** - * Records where escapes occurred in the input document. This is optimal - * for the case where there are few/no escapes. - * - *

This implementation can handle arbitrary length differences between - * the escape and its translation, provided the translation is always smaller - * than the escape. - * - C++ translates newline escapes (1 or 2 chars) to zero chars (an important corner case) - * - Java translates arbitrary-length unicode escapes (>= 6 chars) to 1 char - * - *

This class is tightly coupled to what {@link EscapeAwareReader} - * does with its buffer. - */ -class EscapeTracker { - - private static final int[] EMPTY = new int[0]; - private static final int RECORD_SIZE = 3; - static final EOFException EOF = new EOFException(); - - /* - * Escapes are encoded as tuples (off, after, invalid) where - * - off is the offset in the source file where the escape occurred - * - after is the index of the char following the escape in the input file - * - invalid is the last offset in the buffer which contains the translated chars (exclusive) - * - * Eg for "_\u00a0_" (translates as "_ _"), the buffer looks like - * [_ u00a0_] - * ^ (off) this char has been replaced with the translated value of the escape - * ^^^^^ (after - invalid) these characters are only present in the input, we jump over them when reading - * ^ (invalid) offset at which to jump to 'after' - * ^ (after) first char after the escape - * ^^^^^^ (after - off) total length of the escape in the input - * - * The escape record is (1,7,2) - * - * When reading the buffer we'll copy two blocks - * * "a " - * * then jump over "u00a0" and copy "b" - * - * In general to read until an escape means reading until its 'invalid' - * field, and once that is reached, jump to off + len. - * - */ - private int[] escapeRecords = EMPTY; - /** Index of the next write in the {@link #escapeRecords}. */ - private int nextFreeIdx = 0; - - - /** - * Calls to this method must occur in source order (ie param - * offsetInInput increases monotonically). - */ - void recordEscape(int offsetInInput, int lengthInInput, int lengthInOutput) { - if (nextFreeIdx + 1 >= escapeRecords.length) { - // add 1 to not stay stuck at zero, needs to remain a multiple of RECORD_SIZE - int[] newOffsets = new int[(escapeRecords.length + 1) * RECORD_SIZE]; - System.arraycopy(escapeRecords, 0, newOffsets, 0, escapeRecords.length); - this.escapeRecords = newOffsets; - } - - escapeRecords[nextFreeIdx++] = offsetInInput; - escapeRecords[nextFreeIdx++] = offsetInInput + lengthInInput; - escapeRecords[nextFreeIdx++] = offsetInInput + lengthInOutput; - } - - int inOff(int idx) { - assert idx < nextFreeIdx; - return escapeRecords[idx]; - } - - int indexAfter(int idx) { - assert idx < nextFreeIdx; - return escapeRecords[idx + 1]; - } - - - int invalidIdx(int idx) { - assert idx < nextFreeIdx; - return escapeRecords[idx + 2]; - } - - int inLen(int idx) { - assert idx < nextFreeIdx; - return indexAfter(idx) - inOff(idx); - } - - /** - * Convert an offset in the translated file into an offset in - * the untranslated input. - */ - int inputOffsetAt(int translatedOffset) { - // basically accumulate the lengths of all escapes occurring before the given translatedOffset - int sum = translatedOffset; - for (int i = 0; i < maxEscape(); i += RECORD_SIZE) { - if (inOff(i) < sum) { - sum += inLen(i); - } else { - break; - } - } - return sum; - } - - int maxEscape() { - return nextFreeIdx; - } - - @Override - public String toString() { - StringBuilder res = new StringBuilder("Escape set {"); - for (int i = 0; i < maxEscape(); i += RECORD_SIZE) { - res.append("(at=").append(inOff(i)) - .append(", inlen=").append(inLen(i)) - .append(", invalidAt=").append(invalidIdx(i)) - .append("), "); - } - - return res.append('}').toString(); - } - - /** Backend for a CharStream. Maintains a current position and a mark. */ - class Cursor { - - /** - * This is the index in buf of the next char to read, it always - * holds that buf[pos] is a valid character. - */ - private int pos; - - /** - * Index in {@link #escapeRecords} of the next escape that occurred - * in the source (this means, the buffer is discontinuous at - * inputOffsets[nextEscape]) - */ - private int nextEscape = 0; - - /** - * This is the current offset in the translated document, it - * is shifted from pos by the total length of the escapes that - * occurred before pos. - */ - private int outOffset = 0; - - /** - * A char buffer, which has discontinuities at the indexes - * identified by the {@link #escapeRecords}. It must hold - * that buf.length is the original source length. - */ - private final Chars buf; - - private int mark = Integer.MAX_VALUE; - private int markEscape; - private int markOutOffset; - - Cursor(Chars buf) { - this.buf = buf; - } - - public char next() throws EOFException { - if (pos == buf.length()) { - throw EOF; - } - char c; - - if (nextEscape < maxEscape() && pos == invalidIdx(nextEscape)) { - int pos = indexAfter(nextEscape); // jump past escape - c = buf.charAt(pos); - this.pos = pos + 1; - this.nextEscape += RECORD_SIZE; - } else { - c = buf.charAt(pos); - pos++; - } - outOffset++; - return c; - } - - - public void backup(int numChars) { - ensureMarked(); - if (numChars > markLength()) { - throw new IllegalArgumentException( - "Cannot backup " + numChars + " chars, only " + markLength() + " are saved"); - } - - outOffset -= numChars; - - if (nextEscape <= 0) { - pos -= numChars; // then there were no escapes before the 'pos' - } else if (numChars == 1) { - // fast path, very common - int esc = nextEscape - RECORD_SIZE; // >= 0 because of condition above - if (indexAfter(esc) == pos) { // jump back over the escape - pos = invalidIdx(esc) - 1; - nextEscape = esc; - } else { - pos--; - } - } else { - int newOff = pos; - for (int i = nextEscape - RECORD_SIZE; i >= 0 && numChars > 0; i -= RECORD_SIZE) { - // aa __|||bb - // ^ invalid - // ^^^ jumped - // ^ inOff - // ^^ translated - - int nc = numChars; - numChars -= newOff - indexAfter(i); - newOff = max(indexAfter(i), newOff - nc); - if (numChars <= 0) { // skip "bb", ie everything after the escape - break; - } - - newOff = invalidIdx(i) - 1; // jump back over the escape ||| - numChars--; - nextEscape = i; - - if (numChars <= 0) { - break; - } - } - if (numChars < 0) { - pos = newOff; // newOff was already clipped - } else { - pos = newOff - numChars; // numChars is the remainder - } - } - } - - public void mark() { - this.mark = pos; - this.markEscape = nextEscape; - this.markOutOffset = outOffset; - } - - public void appendMarkSuffix(StringBuilder sb, int suffixLen) { - ensureMarked(); - assert suffixLen <= markLength(); - - - if (markEscape == nextEscape) { - // no escape in the marked range - buf.appendChars(sb, pos - suffixLen, suffixLen); - } else { - // fallback inefficient implementation - getMarkImage().appendChars(sb, markLength() - suffixLen, suffixLen); - } - } - - public Chars getMarkImage() { - if (markEscape == nextEscape) { - // no escape in the marked range, this is the fast path - return buf.slice(mark, markLength()); - } - - StringBuilder sb = new StringBuilder(markLength()); - - int cur = mark; - int esc = markEscape; - while (cur < pos && esc < nextEscape) { - buf.appendChars(sb, cur, invalidIdx(esc) - cur); - cur = indexAfter(esc); - esc += RECORD_SIZE; - } - // no more escape in the range, append everything until the pos - buf.appendChars(sb, cur, pos - cur); - assert sb.length() == markLength() : sb + " should have length " + markLength(); - return Chars.wrap(sb, true); - } - - private void ensureMarked() { - if (mark == Integer.MAX_VALUE) { - throw new IllegalStateException("Mark is not set"); - } - assert mark <= pos : "Wrong mark"; - assert markEscape <= nextEscape : "Wrong mark"; - assert markEscape <= escapeRecords.length : "Wrong escape mark"; - } - - public int curOutOffset() { - return outOffset; - } - - public int markOutOffset() { - return markOutOffset; - } - - public int markLength() { - return outOffset - markOutOffset; - } - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java new file mode 100644 index 0000000000..8b16492ad6 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -0,0 +1,73 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc.io; + + +import net.sourceforge.pmd.lang.ast.impl.javacc.io.FragmentedDocCursor.Fragment; +import net.sourceforge.pmd.util.document.Chars; + +final class FragmentedDocBuilder { + + private final Chars mainBuf; + + private Fragment lastFragment; + private Fragment firstFragment; + + private int curOffInInput; + + FragmentedDocBuilder(Chars buffer) { + this.mainBuf = buffer; + } + + /** + * Calls to this method must occur in source order (ie param + * offsetInInput increases monotonically). + */ + void recordDelta(int startInInput, int endInInput, Chars translation) { + int inLength = endInInput - startInInput; + if (firstFragment == null) { + assert lastFragment == null; + firstFragment = new Fragment(null, startInInput, mainBuf.slice(0, startInInput)); + lastFragment = new Fragment(firstFragment, inLength, translation); + curOffInInput = endInInput; + return; + } + + Fragment last = lastFragment; + int prevLen = startInInput - curOffInInput; + last = new Fragment(last, prevLen, mainBuf.slice(curOffInInput, prevLen)); + last = new Fragment(last, inLength, translation); + this.lastFragment = last; + this.curOffInInput = endInInput; + } + + FragmentedDocCursor newCursor() { + if (firstFragment == null) { // no deltas in whole document + Fragment whole = new Fragment(null, mainBuf.length(), mainBuf); + return new FragmentedDocCursor(whole); + } else { + if (curOffInInput < mainBuf.length()) { + int remLen = mainBuf.length() - curOffInInput; + Chars remainder = mainBuf.slice(curOffInInput, remLen); + lastFragment = new Fragment(lastFragment, remLen, remainder); + } + + return new FragmentedDocCursor(firstFragment); + } + } + + int inputOffsetAt(int outputOffset) { + Fragment f = firstFragment; + if (f == null) { + return outputOffset; + } + int sum = outputOffset; + while (f != null && f.inStart() < sum) { + sum += f.inLen(); + f = f.next; + } + return sum; + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java new file mode 100644 index 0000000000..95cb9b0c52 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -0,0 +1,172 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc.io; + +import java.io.EOFException; + +import net.sourceforge.pmd.util.document.Chars; + +final class FragmentedDocCursor { + + private Fragment cur; + private int curOutPos; + + private Fragment mark; + private int markOutPos; + + FragmentedDocCursor(Fragment first) { + cur = first; + mark = first; + } + + + char next() throws EOFException { + Fragment f = cur; + + while (f != null && curOutPos >= f.outEnd()) { + f = f.next; + } + if (f == null) { + throw new EOFException(); + } + + cur = f; + return f.charAt(curOutPos++); + } + + void backup(final int amount) { + Fragment f = cur; + + int a = amount; + while (f != null && a > 0) { + if (a <= f.outLen()) { + break; + } else { + a -= f.outLen(); + f = f.prev; + } + } + + if (f == null) { + throw new IllegalArgumentException("Cannot backup by " + amount + " chars"); + } + + this.curOutPos = curOutPos - amount; + this.cur = f; + } + + void mark() { + mark = cur; + markOutPos = curOutPos; + } + + int markInOffset() { + return mark.outToIn(markOutPos); + } + + int curInOffset() { + return cur.outToIn(curOutPos); + } + + int markLength() { + return curOutPos - markOutPos; + } + + + public void appendMarkSuffix(StringBuilder sb, int suffixLen) { + assert suffixLen <= markLength(); + + if (cur == mark) { + // no escape in the marked range + cur.chars.appendChars(sb, curOutPos - cur.outStart - suffixLen, suffixLen); + } else { + // fallback inefficient implementation + getMarkImage().appendChars(sb, markLength() - suffixLen, suffixLen); + } + } + + public Chars getMarkImage() { + Fragment f = mark; + if (f == cur) { // same fragment, this is the fast path + return f.chars.slice(markOutPos - f.outStart(), markLength()); + } + + StringBuilder sb = new StringBuilder(markLength()); + + f.appendAbs(sb, markOutPos, f.outEnd()); + f = f.next; + while (f != cur) { + f.appendAbs(sb, f.outStart(), f.outEnd()); + f = f.next; + } + f.appendAbs(sb, f.outStart(), curOutPos); + assert sb.length() == markLength() : sb + " should have length " + markLength(); + return Chars.wrap(sb, true); + } + + + static class Fragment { + + private final Chars chars; + + private final Fragment prev; + Fragment next; + + private final int outStart; + private final int inStart; + private final int inLength; + + Fragment(Fragment prev, int inLength, Chars chars) { + this.chars = chars; + this.prev = prev; + this.inLength = inLength; + if (prev != null) { + prev.next = this; + this.outStart = prev.outEnd(); + this.inStart = prev.inStart + prev.inLen(); + } else { + this.outStart = this.inStart = 0; + } + } + + + void appendAbs(StringBuilder sb, int absOffset, int absEndOffset) { + chars.appendChars(sb, absOffset - outStart, absEndOffset - outStart); + } + + + char charAt(int absPos) { + return chars.charAt(absPos - outStart); + } + + int outStart() { + return outStart; + } + + int outLen() { + return chars.length(); + } + + int outEnd() { + return outStart + outLen(); + } + + int inStart() { + return inStart; + } + + int inLen() { + return inLength; + } + + int inEnd() { + return inStart + inLength; + } + + int outToIn(int outOffset) { + return inStart() + (outOffset - outStart()); + } + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index db9e49d203..2e34649d66 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -35,14 +35,15 @@ public final class JavaEscapeReader extends BackslashEscapeReader { // consume all the 'u's off++; } - int end = replaceFirstBackslashWithEscape(firstBackslashOff, off - 1); - return recordEscape(firstBackslashOff, end - firstBackslashOff, 1); + Chars value = escapeValue(firstBackslashOff, off - 1); + int endOffset = off + 4; // + 4 hex digits + return recordEscape(firstBackslashOff, endOffset, value); } else { return abortEscape(off, maxOff); } } - private int replaceFirstBackslashWithEscape(int posOfFirstBackSlash, int offOfTheU) throws IOException { + private Chars escapeValue(int posOfFirstBackSlash, int offOfTheU) throws IOException { try { char c = (char) ( hexVal(input.charAt(++offOfTheU)) << 12 @@ -50,8 +51,8 @@ public final class JavaEscapeReader extends BackslashEscapeReader { | hexVal(input.charAt(++offOfTheU)) << 4 | hexVal(input.charAt(++offOfTheU)) ); - input.set(posOfFirstBackSlash, c); // replace the start char of the backslash - return offOfTheU + 1; + + return Chars.wrap(new char[] { c }, true); } catch (NumberFormatException | IndexOutOfBoundsException e) { String message = "Invalid escape sequence at line " + getLine(posOfFirstBackSlash) + ", column " diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index 660df96ca5..fe336ffddb 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -159,7 +159,6 @@ public class CharStreamImplTest { assertEquals('d', stream.readChar()); expect.expect(IllegalArgumentException.class); - expect.expectMessage("only 2 are saved"); stream.backup(10); } @Test @@ -173,7 +172,6 @@ public class CharStreamImplTest { assertEquals('d', stream.readChar()); expect.expect(IllegalArgumentException.class); - expect.expectMessage("only 4 are saved"); stream.backup(10); } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java index e0531eca21..b041556f53 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -21,10 +21,10 @@ public class CppEscapeReader extends BackslashEscapeReader { int off = backSlashOff; if (input.charAt(off) == NEWLINE) { - return recordEscape(backSlashOff, 2, 0); + return recordEscape(backSlashOff, 2, Chars.EMPTY); } else if (input.charAt(off) == CARRIAGE_RETURN) { if (input.charAt(++off) == NEWLINE) { - return recordEscape(backSlashOff, 3, 0); + return recordEscape(backSlashOff, 3, Chars.EMPTY); } } From 1210d6864068064830c1a2cb37a42f129b335f32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 19 Apr 2020 19:36:54 +0200 Subject: [PATCH 023/136] Remove mutable char buffers --- .../ast/impl/javacc/io/FragmentedDocCursor.java | 14 ++++++++++++-- .../lang/ast/impl/javacc/io/JavaEscapeReader.java | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 95cb9b0c52..6ed2dfbf95 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -125,7 +125,7 @@ final class FragmentedDocCursor { if (prev != null) { prev.next = this; this.outStart = prev.outEnd(); - this.inStart = prev.inStart + prev.inLen(); + this.inStart = prev.inEnd(); } else { this.outStart = this.inStart = 0; } @@ -133,7 +133,7 @@ final class FragmentedDocCursor { void appendAbs(StringBuilder sb, int absOffset, int absEndOffset) { - chars.appendChars(sb, absOffset - outStart, absEndOffset - outStart); + chars.appendChars(sb, absOffset - outStart, absEndOffset - absOffset); } @@ -168,5 +168,15 @@ final class FragmentedDocCursor { int outToIn(int outOffset) { return inStart() + (outOffset - outStart()); } + + @Override + public String toString() { + return "Fragment{" + + "chars=" + chars + + ", outStart=" + outStart + + ", inStart=" + inStart + + ", inLength=" + inLength + + '}'; + } } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index 2e34649d66..ea54cb9b7c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -52,7 +52,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { | hexVal(input.charAt(++offOfTheU)) ); - return Chars.wrap(new char[] { c }, true); + return Chars.wrap(new char[] { c }); } catch (NumberFormatException | IndexOutOfBoundsException e) { String message = "Invalid escape sequence at line " + getLine(posOfFirstBackSlash) + ", column " From 5ab241ecc5bfd4c69a456d7e64816652fb9ca64d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 19 Apr 2020 20:37:11 +0200 Subject: [PATCH 024/136] Use string internally This is to profit from compact string features, and to make the use of StringBuilders more efficient. --- .../impl/javacc/io/FragmentedDocBuilder.java | 4 ++- .../impl/javacc/io/FragmentedDocCursor.java | 9 +++-- .../ast/impl/javacc/io/JavaEscapeReader.java | 2 +- .../impl/javacc/io/JavaEscapeReaderTest.java | 2 +- .../pmd/lang/java/ast/ParserCornersTest.java | 34 +++++++++++++++---- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 8b16492ad6..2cd2cfa7aa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -37,7 +37,9 @@ final class FragmentedDocBuilder { Fragment last = lastFragment; int prevLen = startInInput - curOffInInput; - last = new Fragment(last, prevLen, mainBuf.slice(curOffInInput, prevLen)); + if (prevLen != 0) { + last = new Fragment(last, prevLen, mainBuf.slice(curOffInInput, prevLen)); + } last = new Fragment(last, inLength, translation); this.lastFragment = last; this.curOffInInput = endInInput; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 6ed2dfbf95..2ee6b910f5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -10,6 +10,8 @@ import net.sourceforge.pmd.util.document.Chars; final class FragmentedDocCursor { + private static final EOFException EOF = new EOFException(); + private Fragment cur; private int curOutPos; @@ -28,8 +30,9 @@ final class FragmentedDocCursor { while (f != null && curOutPos >= f.outEnd()) { f = f.next; } + if (f == null) { - throw new EOFException(); + throw EOF; } cur = f; @@ -103,11 +106,11 @@ final class FragmentedDocCursor { } f.appendAbs(sb, f.outStart(), curOutPos); assert sb.length() == markLength() : sb + " should have length " + markLength(); - return Chars.wrap(sb, true); + return Chars.wrap(sb); } - static class Fragment { + static final class Fragment { private final Chars chars; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index ea54cb9b7c..2a6b2d151f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -52,7 +52,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { | hexVal(input.charAt(++offOfTheU)) ); - return Chars.wrap(new char[] { c }); + return Chars.wrap(Character.toString(c)); } catch (NumberFormatException | IndexOutOfBoundsException e) { String message = "Invalid escape sequence at line " + getLine(posOfFirstBackSlash) + ", column " diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index 492afc7f06..9443514679 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -17,7 +17,7 @@ public class JavaEscapeReaderTest { @NonNull public JavaEscapeReader readString(String input) { - return new JavaEscapeReader(Chars.wrap(input, true)); + return new JavaEscapeReader(Chars.wrap(input)); } diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java index 1180d2645b..228dc8a608 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java @@ -91,12 +91,34 @@ public class ParserCornersTest extends BaseJavaTreeDumpTest { @Test public void testUnicodeEscapes2() { java.parse("\n" - + "public final class TimeZoneNames_zh_TW extends TimeZoneNamesBundle {\n" - + "\n" - + " String ACT[] = new String[] {\"Acre \\u6642\\u9593\", \"ACT\",\n" - + " \"Acre \\u590f\\u4ee4\\u6642\\u9593\", \"ACST\",\n" - + " \"Acre \\u6642\\u9593\", \"ACT\"};" - + "}"); + + "public final class TimeZoneNames_zh_TW extends TimeZoneNamesBundle {\n" + + "\n" + + " String ACT[] = new String[] {\"Acre \\u6642\\u9593\", \"ACT\",\n" + + " \"Acre \\u590f\\u4ee4\\u6642\\u9593\", \"ACST\",\n" + + " \"Acre \\u6642\\u9593\", \"ACT\"};" + + "}"); + } + + @Test + public void testUnicodeEscapesInComment() { + java.parse("class Foo {" + + "\n" + + " /**\n" + + " * The constant value of this field is the smallest value of type\n" + + " * {@code char}, {@code '\\u005Cu0000'}.\n" + + " *\n" + + " * @since 1.0.2\n" + + " */\n" + + " public static final char MIN_VALUE = '\\u0000';\n" + + "\n" + + " /**\n" + + " * The constant value of this field is the largest value of type\n" + + " * {@code char}, {@code '\\u005CuFFFF'}.\n" + + " *\n" + + " * @since 1.0.2\n" + + " */\n" + + " public static final char MAX_VALUE = '\\uFFFF';" + + "}"); } @Test From 3170fcf3c4641a47c00bdd3bd2f53f336005c744 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 19 Apr 2020 22:42:30 +0200 Subject: [PATCH 025/136] Optimal impl for appendMarkSuffix --- .../impl/javacc/io/FragmentedDocBuilder.java | 24 +++++-- .../impl/javacc/io/FragmentedDocCursor.java | 65 ++++++++++++------- 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 2cd2cfa7aa..7bd24d38af 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -22,10 +22,19 @@ final class FragmentedDocBuilder { } /** - * Calls to this method must occur in source order (ie param - * offsetInInput increases monotonically). + * Add a new fragment. + * + * @param startInInput Start (inclusive) of the overwritten text in the source + * @param endInInput End (exclusive) ... + * @param translation Characters with which the range startInInput..endInInput are overwritten. + * This may be empty. */ void recordDelta(int startInInput, int endInInput, Chars translation) { + assert curOffInInput <= startInInput + : "Already moved past " + curOffInInput + ", cannot add delta at " + startInInput; + assert startInInput <= endInInput : "Offsets must be ordered"; + assert translation != null : "Translation cannot be null"; + int inLength = endInInput - startInInput; if (firstFragment == null) { assert lastFragment == null; @@ -45,17 +54,20 @@ final class FragmentedDocBuilder { this.curOffInInput = endInInput; } + /** + * Finalize the construction process. + */ FragmentedDocCursor newCursor() { - if (firstFragment == null) { // no deltas in whole document - Fragment whole = new Fragment(null, mainBuf.length(), mainBuf); - return new FragmentedDocCursor(whole); + if (firstFragment == null) { + // no deltas in whole document, there's a single fragment + return new FragmentedDocCursor(new Fragment(null, mainBuf.length(), mainBuf)); } else { if (curOffInInput < mainBuf.length()) { + // there's some text left between the last fragment and the end of the doc int remLen = mainBuf.length() - curOffInInput; Chars remainder = mainBuf.slice(curOffInInput, remLen); lastFragment = new Fragment(lastFragment, remLen, remainder); } - return new FragmentedDocCursor(firstFragment); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 2ee6b910f5..91b24dcec9 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -6,6 +6,8 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.EOFException; +import org.checkerframework.checker.nullness.qual.Nullable; + import net.sourceforge.pmd.util.document.Chars; final class FragmentedDocCursor { @@ -18,9 +20,9 @@ final class FragmentedDocCursor { private Fragment mark; private int markOutPos; - FragmentedDocCursor(Fragment first) { - cur = first; - mark = first; + FragmentedDocCursor(Fragment firstFragment) { + cur = firstFragment; + mark = firstFragment; } @@ -78,18 +80,36 @@ final class FragmentedDocCursor { } - public void appendMarkSuffix(StringBuilder sb, int suffixLen) { + public void appendMarkSuffix(StringBuilder sb, final int suffixLen) { assert suffixLen <= markLength(); if (cur == mark) { - // no escape in the marked range - cur.chars.appendChars(sb, curOutPos - cur.outStart - suffixLen, suffixLen); + // entire mark is in a single fragment, fast path 1 + appendSuffix(sb, suffixLen); } else { - // fallback inefficient implementation - getMarkImage().appendChars(sb, markLength() - suffixLen, suffixLen); + + // look backwards until we find the fragment that starts the suffix + Fragment f = cur; + int suffixStart = curOutPos - suffixLen; + while (f != null && f.outStart() > suffixStart) { + f = f.prev; + } + assert f != null; + + if (f == cur) { + // entire suffix is in a single fragment, fast path 2 + appendSuffix(sb, suffixLen); + } + + sb.ensureCapacity(sb.length() + suffixLen); + appendUntilCurPos(f, sb, suffixStart); } } + public void appendSuffix(StringBuilder sb, int suffixLen) { + cur.chars.appendChars(sb, curOutPos - cur.outStart - suffixLen, suffixLen); + } + public Chars getMarkImage() { Fragment f = mark; if (f == cur) { // same fragment, this is the fast path @@ -97,16 +117,24 @@ final class FragmentedDocCursor { } StringBuilder sb = new StringBuilder(markLength()); + appendUntilCurPos(f, sb, markOutPos); + assert sb.length() == markLength() : sb + " should have length " + markLength(); + return Chars.wrap(sb); + } - f.appendAbs(sb, markOutPos, f.outEnd()); + public void appendUntilCurPos(Fragment f, StringBuilder sb, int startOutPos) { + assert f.outStart() <= startOutPos && startOutPos < f.outEnd(); + + // append the suffix of the first fragment after the start pos + f.appendAbs(sb, startOutPos, f.outEnd()); f = f.next; while (f != cur) { + // append whole fragments f.appendAbs(sb, f.outStart(), f.outEnd()); f = f.next; } + // append the prefix of the last fragment until the current pos f.appendAbs(sb, f.outStart(), curOutPos); - assert sb.length() == markLength() : sb + " should have length " + markLength(); - return Chars.wrap(sb); } @@ -114,14 +142,14 @@ final class FragmentedDocCursor { private final Chars chars; - private final Fragment prev; - Fragment next; + final @Nullable Fragment prev; + @Nullable Fragment next; private final int outStart; private final int inStart; private final int inLength; - Fragment(Fragment prev, int inLength, Chars chars) { + Fragment(@Nullable Fragment prev, int inLength, Chars chars) { this.chars = chars; this.prev = prev; this.inLength = inLength; @@ -134,12 +162,10 @@ final class FragmentedDocCursor { } } - void appendAbs(StringBuilder sb, int absOffset, int absEndOffset) { chars.appendChars(sb, absOffset - outStart, absEndOffset - absOffset); } - char charAt(int absPos) { return chars.charAt(absPos - outStart); } @@ -174,12 +200,7 @@ final class FragmentedDocCursor { @Override public String toString() { - return "Fragment{" - + "chars=" + chars - + ", outStart=" + outStart - + ", inStart=" + inStart - + ", inLength=" + inLength - + '}'; + return "Fragment[" + inStart + ".." + outStart + "]\n" + chars; } } } From 02684accba62fbfd046a3095cefaecc24cf38913 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 19 Apr 2020 22:47:30 +0200 Subject: [PATCH 026/136] PICKME --- .../impl/javacc/io/FragmentedDocBuilder.java | 3 +- .../impl/javacc/io/FragmentedDocCursor.java | 140 ++++++++---------- 2 files changed, 66 insertions(+), 77 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 7bd24d38af..8fdb44cb67 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -59,7 +59,8 @@ final class FragmentedDocBuilder { */ FragmentedDocCursor newCursor() { if (firstFragment == null) { - // no deltas in whole document, there's a single fragment + // No deltas in whole document, there's a single fragment + // This is the case for > 97% of Java files (source: OpenJDK) return new FragmentedDocCursor(new Fragment(null, mainBuf.length(), mainBuf)); } else { if (curOffInInput < mainBuf.length()) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 91b24dcec9..66e414cd2b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -5,7 +5,9 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.EOFException; +import java.util.Objects; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.util.document.Chars; @@ -23,6 +25,7 @@ final class FragmentedDocCursor { FragmentedDocCursor(Fragment firstFragment) { cur = firstFragment; mark = firstFragment; + checkAssertions(); } @@ -30,7 +33,7 @@ final class FragmentedDocCursor { Fragment f = cur; while (f != null && curOutPos >= f.outEnd()) { - f = f.next; + f = f.next; // this is a loop to handle chained zero-length fragments } if (f == null) { @@ -42,24 +45,68 @@ final class FragmentedDocCursor { } void backup(final int amount) { + final int posToRetreatTo = curOutPos - amount; + this.curOutPos = posToRetreatTo; + this.cur = findBackwards(posToRetreatTo); + checkAssertions(); + } + + + void appendMarkSuffix(StringBuilder sb, final int suffixLen) { + assert suffixLen <= markLength() : "Suffix is greater than the mark length? " + suffixLen + " > " + markLength(); + + if (cur == mark || cur.outStart() <= curOutPos - suffixLen) { + // entire suffix is in the last fragment, fast path + cur.chars.appendChars(sb, curOutPos - cur.outStart - suffixLen, suffixLen); + } else { + int suffixStart = curOutPos - suffixLen; + Fragment f = findBackwards(suffixStart); + sb.ensureCapacity(sb.length() + suffixLen); + appendUntilCurPos(f, sb, suffixStart); + } + } + + Chars getMarkImage() { + Fragment f = mark; + if (f == cur) { // same fragment, this is the fast path + return f.chars.slice(markOutPos - f.outStart(), markLength()); + } + + StringBuilder sb = new StringBuilder(markLength()); + appendUntilCurPos(f, sb, markOutPos); + assert sb.length() == markLength() : sb + " should have length " + markLength(); + return Chars.wrap(sb); + } + + private void appendUntilCurPos(Fragment f, StringBuilder sb, int startOutPos) { + assert f != null && f != cur; // this won't work otherwise + assert f.outStart() <= startOutPos && startOutPos < f.outEnd(); + + // append the suffix of the first fragment after the start pos + f.appendAbs(sb, startOutPos, f.outEnd()); + f = f.next; + while (f != cur) { + // append whole fragments + f.appendAbs(sb, f.outStart(), f.outEnd()); + f = f.next; + } + // append the prefix of the last fragment until the current pos + f.appendAbs(sb, f.outStart(), curOutPos); + } + + + private void checkAssertions() { + assert mark != null && cur != null : "Null mark or current fragment"; + assert cur.outStart() >= mark.outStart() : "Mark is after the current fragment"; + } + + // find the fragment that contains the given out offset + private @NonNull Fragment findBackwards(int posToRetreatTo) { Fragment f = cur; - - int a = amount; - while (f != null && a > 0) { - if (a <= f.outLen()) { - break; - } else { - a -= f.outLen(); - f = f.prev; - } + while (f != null && f.outStart() > posToRetreatTo) { + f = f.prev; } - - if (f == null) { - throw new IllegalArgumentException("Cannot backup by " + amount + " chars"); - } - - this.curOutPos = curOutPos - amount; - this.cur = f; + return Objects.requireNonNull(f, "Cannot retreat to " + posToRetreatTo); } void mark() { @@ -79,65 +126,6 @@ final class FragmentedDocCursor { return curOutPos - markOutPos; } - - public void appendMarkSuffix(StringBuilder sb, final int suffixLen) { - assert suffixLen <= markLength(); - - if (cur == mark) { - // entire mark is in a single fragment, fast path 1 - appendSuffix(sb, suffixLen); - } else { - - // look backwards until we find the fragment that starts the suffix - Fragment f = cur; - int suffixStart = curOutPos - suffixLen; - while (f != null && f.outStart() > suffixStart) { - f = f.prev; - } - assert f != null; - - if (f == cur) { - // entire suffix is in a single fragment, fast path 2 - appendSuffix(sb, suffixLen); - } - - sb.ensureCapacity(sb.length() + suffixLen); - appendUntilCurPos(f, sb, suffixStart); - } - } - - public void appendSuffix(StringBuilder sb, int suffixLen) { - cur.chars.appendChars(sb, curOutPos - cur.outStart - suffixLen, suffixLen); - } - - public Chars getMarkImage() { - Fragment f = mark; - if (f == cur) { // same fragment, this is the fast path - return f.chars.slice(markOutPos - f.outStart(), markLength()); - } - - StringBuilder sb = new StringBuilder(markLength()); - appendUntilCurPos(f, sb, markOutPos); - assert sb.length() == markLength() : sb + " should have length " + markLength(); - return Chars.wrap(sb); - } - - public void appendUntilCurPos(Fragment f, StringBuilder sb, int startOutPos) { - assert f.outStart() <= startOutPos && startOutPos < f.outEnd(); - - // append the suffix of the first fragment after the start pos - f.appendAbs(sb, startOutPos, f.outEnd()); - f = f.next; - while (f != cur) { - // append whole fragments - f.appendAbs(sb, f.outStart(), f.outEnd()); - f = f.next; - } - // append the prefix of the last fragment until the current pos - f.appendAbs(sb, f.outStart(), curOutPos); - } - - static final class Fragment { private final Chars chars; From db701961595214daa2ba3e6da0c3ebd7f579871f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 19 Apr 2020 23:59:12 +0200 Subject: [PATCH 027/136] Ignore MORE token accumulation --- .../pmd/lang/ast/impl/javacc/JavaccTokenDocument.java | 10 ++++++++++ .../pmd/lang/ast/impl/javacc/io/CharStream.java | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 9529abf521..956329b2ff 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -25,6 +25,16 @@ public class JavaccTokenDocument extends TokenDocument { super(textDocument); } + /** + * Returns true if the lexer should accumulate the image of MORE + * tokens into the StringBuilder jjimage. This is useless in our + * current implementations. The default returns false, which makes + * {@link CharStream#appendSuffix(StringBuilder, int)} a noop. + */ + public boolean useMarkSuffix() { + return false; + } + /** * Create new (possibly) escaping reader for the given text. The default * implementation doesn't do any escaping. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index c52699936a..64421a3bbf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -20,10 +20,12 @@ public final class CharStream { private final JavaccTokenDocument tokenDoc; private final FragmentedDocCursor cursor; + private final boolean useMarkSuffix; private CharStream(JavaccTokenDocument tokenDoc, FragmentedDocCursor cursor) { this.tokenDoc = tokenDoc; this.cursor = cursor; + useMarkSuffix = tokenDoc.useMarkSuffix(); } /** @@ -87,7 +89,9 @@ public final class CharStream { * @throws IndexOutOfBoundsException If len is greater than the length of the current token */ public void appendSuffix(StringBuilder sb, int len) { - cursor.appendMarkSuffix(sb, len); + if (useMarkSuffix) { + cursor.appendMarkSuffix(sb, len); + } // otherwise dead code, kept because Javacc's argument expressions do side effects } From 01468cd733a6d82be753726c561d49d265cfef58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 20 Apr 2020 01:07:51 +0200 Subject: [PATCH 028/136] Reuse stringbuilders --- .../impl/javacc/io/FragmentedDocCursor.java | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 66e414cd2b..85eca6c3f3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -5,7 +5,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.EOFException; -import java.util.Objects; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -22,12 +21,28 @@ final class FragmentedDocCursor { private Fragment mark; private int markOutPos; + // We can reuse a stringbuilder + // This means, the byte[] buffer is shared, and doesn't need to be + // resized so often. StringBuilder::toString always does an array + // copy anyway. + + // This also means that if we encounter some non-latin1 characters, + // all the following tokens will be coded as utf16. The string + // builder is only used for tokens that span several file fragments + // though, so that is a rare situation + private final StringBuilder strBuilder = new StringBuilder(); + FragmentedDocCursor(Fragment firstFragment) { cur = firstFragment; mark = firstFragment; checkAssertions(); } + private StringBuilder freshStrBuilder(int minCap) { + strBuilder.setLength(0); + strBuilder.ensureCapacity(minCap); + return strBuilder; + } char next() throws EOFException { Fragment f = cur; @@ -72,7 +87,7 @@ final class FragmentedDocCursor { return f.chars.slice(markOutPos - f.outStart(), markLength()); } - StringBuilder sb = new StringBuilder(markLength()); + StringBuilder sb = freshStrBuilder(markLength()); appendUntilCurPos(f, sb, markOutPos); assert sb.length() == markLength() : sb + " should have length " + markLength(); return Chars.wrap(sb); @@ -106,7 +121,10 @@ final class FragmentedDocCursor { while (f != null && f.outStart() > posToRetreatTo) { f = f.prev; } - return Objects.requireNonNull(f, "Cannot retreat to " + posToRetreatTo); + if (f == null) { + throw new IllegalArgumentException("Cannot retreat to " + posToRetreatTo); + } + return f; } void mark() { From 6064d16ecc919f426bd1937e87bae9a55a70fc5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 20 Apr 2020 03:43:58 +0200 Subject: [PATCH 029/136] Add fast paths --- .../impl/javacc/io/FragmentedDocCursor.java | 23 +++++++++++++------ .../ast/impl/javacc/io/JavaEscapeReader.java | 2 +- .../pmd/lang/java/ast/ParserCornersTest.java | 2 +- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 85eca6c3f3..500f01460c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -47,19 +47,28 @@ final class FragmentedDocCursor { char next() throws EOFException { Fragment f = cur; - while (f != null && curOutPos >= f.outEnd()) { - f = f.next; // this is a loop to handle chained zero-length fragments + if (curOutPos >= f.outEnd()) { + // slow path, fragment transition + do { + f = f.next; + } while (f != null && curOutPos >= f.outEnd()); + + if (f == null) { + throw EOF; + } + cur = f; } - if (f == null) { - throw EOF; - } - - cur = f; return f.charAt(curOutPos++); } void backup(final int amount) { + if (amount == 1 && cur.outStart() < curOutPos) { + // fast path + curOutPos--; + return; + } + final int posToRetreatTo = curOutPos - amount; this.curOutPos = posToRetreatTo; this.cur = findBackwards(posToRetreatTo); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index 2a6b2d151f..3ae12bb84e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -30,7 +30,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { // is there an escape at offset firstBslashOff? if ((bslashCount & 1) == 1 // odd number of backslashes && off < input.length() && input.charAt(off) == 'u') { // at least one 'u' - // odd number of backslashes, this is enough to expect an escape or throw an exception + // this is enough to expect an escape or throw an exception while (off < input.length() && input.charAt(off) == 'u') { // consume all the 'u's off++; diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java index 228dc8a608..2fedde03b3 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java @@ -113,7 +113,7 @@ public class ParserCornersTest extends BaseJavaTreeDumpTest { + "\n" + " /**\n" + " * The constant value of this field is the largest value of type\n" - + " * {@code char}, {@code '\\u005CuFFFF'}.\n" + + " * {@code char}, {@code '\\u005C\\uFFFF'}.\n" + " *\n" + " * @since 1.0.2\n" + " */\n" From 1c4b241255233afe1974dbf982d79eb8e44770ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 20 Apr 2020 06:57:48 +0200 Subject: [PATCH 030/136] Pool some strings --- .../main/java/net/sourceforge/pmd/PMD.java | 4 + .../pmd/lang/ast/impl/javacc/JavaccToken.java | 4 - .../ast/impl/javacc/JavaccTokenDocument.java | 14 ++ .../pmd/lang/ast/impl/javacc/StringPool.java | 125 ++++++++++++++++++ .../pmd/lang/java/ast/JavaTokenDocument.java | 6 + 5 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java index e24bdbfe5c..ec7476f730 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java @@ -32,6 +32,10 @@ import net.sourceforge.pmd.cli.PMDParameters; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.lang.LanguageVersionDiscoverer; +import net.sourceforge.pmd.lang.LanguageVersionHandler; +import net.sourceforge.pmd.lang.Parser; +import net.sourceforge.pmd.lang.ParserOptions; +import net.sourceforge.pmd.lang.ast.impl.javacc.StringPool; import net.sourceforge.pmd.processor.AbstractPMDProcessor; import net.sourceforge.pmd.renderers.Renderer; import net.sourceforge.pmd.reporting.GlobalAnalysisListener; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index cd4f087df9..866ba74f3f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -142,10 +142,6 @@ public class JavaccToken implements GenericToken { return image; } - public boolean imageEquals(CharSequence charSequence) { - return StringUtils.equals(image, charSequence); - } - @Override public TextRegion getRegion() { return TextRegion.fromBothOffsets(startOffset, endOffset); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 956329b2ff..21b023a72e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -19,6 +19,8 @@ import net.sourceforge.pmd.util.document.TextDocument; */ public class JavaccTokenDocument extends TokenDocument { + final StringPool stringPool = new StringPool(); + private JavaccToken first; public JavaccTokenDocument(TextDocument textDocument) { @@ -74,6 +76,18 @@ public class JavaccTokenDocument extends TokenDocument { return first.next; } + final String computeImage(JavaccToken t) { + CharSequence imageCs = t.getImageCs(); + if (imageCs instanceof String) { + return (String) imageCs; + } + return stringPool.toString(imageCs, isImagePooled(t)); + } + + protected boolean isImagePooled(JavaccToken t) { + return false; + } + /** * Returns a string that describes the token kind. * diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java new file mode 100644 index 0000000000..7a0d1e464c --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java @@ -0,0 +1,125 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc; + + +import java.util.HashMap; +import java.util.LongSummaryStatistics; +import java.util.Map; + +import net.sourceforge.pmd.util.document.Chars; + +/** + * Simple pooling mechanism. Two {@link Chars} are equal if their + * charsequence is equal. Note that most {@link Chars} instances + * backing tokens hold a strong reference to the file's entire string + * representation, so a pool must not be reused between several files. + */ +public final class StringPool { + + private static final boolean COLLECT_STATS = false; + private static final Stats stats = new Stats(); + + private static final String[] SINGLE_CHARS; + + + static { + // all ascii characters + SINGLE_CHARS = new String[128]; + for (char i = 0; i < 128; i++) { + SINGLE_CHARS[i] = String.valueOf(i); + } + } + + + private final Map pool = new HashMap<>(); + + String toString(CharSequence c, boolean doPool) { + if (c.length() == 1) { + char fst = c.charAt(0); + if (fst < 128) { + return SINGLE_CHARS[fst]; + } + } + + if (doPool && c instanceof Chars) { + if (COLLECT_STATS) { + return pool.compute((Chars) c, + (chars, s) -> { + if (s != null) { + stats.addCacheHit(s.length()); + return s; + } else { + stats.addCacheMiss(chars.length()); + return chars.toString(); + } + }); + } else { + return pool.computeIfAbsent((Chars) c, Chars::toString); + } + } + if (COLLECT_STATS) { + stats.addPass(c); + } + return c.toString(); + } + + public static void printStats() { + stats.print(); + } + + private static class Stats { + + long hits; + long miss; + final LongSummaryStatistics poolContents = new LongSummaryStatistics(); + long hitLen; + long totalTotal; + + long notPooledAlloc; + + public void print() { + System.err.println("String pool stats"); + System.err.println("================="); + System.err.println("Hits: " + hits + " (" + (hits * 100 / (total() + 1)) + "% of " + total() + ")"); + System.err.println("Hit length (net savings): " + hitLen + " " + toSize(hitLen)); + System.err.println( + "Total pool size: " + poolContents.getCount() + " strings (" + toSize(poolContents.getSum()) + ")"); + System.err.println("Avg pooled string length: " + poolContents.getAverage() + " chars"); + System.err.println("Unpooled string length: " + notPooledAlloc + " chars (" + toSize(notPooledAlloc) + ")"); + } + + private String toSize(long charLen) { + // assuming all pooled chars are latin-1, so 1B in a compressed string (byte[], not char[]) + if (charLen > (1 << 20)) { + return (charLen >> 20) + " MB"; + } else if (charLen > (1 << 10)) { + return (charLen >> 10) + " kB"; + } + return charLen + " B"; + } + + void addCacheMiss(int len) { + miss++; + poolContents.accept(len); + } + + void addCacheHit(int len) { + hits++; + hitLen += len; + } + + long total() { + return hits + miss; + } + + void addPass(CharSequence c) { + totalTotal++; + if (!(c instanceof String)) { + notPooledAlloc += c.length(); + } + } + } +} diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 1c62d083c0..56f8690c06 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.lang.java.ast; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.FORMAL_COMMENT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.GT; +import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.IDENTIFIER; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.MULTI_LINE_COMMENT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.RSIGNEDSHIFT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.RUNSIGNEDSHIFT; @@ -50,6 +51,11 @@ final class JavaTokenDocument extends JavaccTokenDocument { return new JavaEscapeReader(text); } + @Override + protected boolean isImagePooled(JavaccToken t) { + return t.kind == IDENTIFIER; + } + @Override protected @Nullable String describeKindImpl(int kind) { return JavaTokenKinds.describe(kind); From 7f65b1f91047cea920c20bc5ee9e7f1cd2f887dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 20 Apr 2020 08:07:04 +0200 Subject: [PATCH 031/136] Improve pooling stats --- .../pmd/lang/ast/impl/javacc/StringPool.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java index 7a0d1e464c..a8a238f473 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java @@ -5,9 +5,9 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import java.util.HashMap; import java.util.LongSummaryStatistics; import java.util.Map; +import java.util.WeakHashMap; import net.sourceforge.pmd.util.document.Chars; @@ -19,12 +19,12 @@ import net.sourceforge.pmd.util.document.Chars; */ public final class StringPool { + // This will be constant-folded by the JIT, use false in production private static final boolean COLLECT_STATS = false; - private static final Stats stats = new Stats(); + private static final Stats STATS = new Stats(); private static final String[] SINGLE_CHARS; - static { // all ascii characters SINGLE_CHARS = new String[128]; @@ -34,12 +34,15 @@ public final class StringPool { } - private final Map pool = new HashMap<>(); + private final Map pool = new WeakHashMap<>(); String toString(CharSequence c, boolean doPool) { if (c.length() == 1) { char fst = c.charAt(0); if (fst < 128) { + if (COLLECT_STATS) { + STATS.addCacheHit(1); + } return SINGLE_CHARS[fst]; } } @@ -49,10 +52,10 @@ public final class StringPool { return pool.compute((Chars) c, (chars, s) -> { if (s != null) { - stats.addCacheHit(s.length()); + STATS.addCacheHit(s.length()); return s; } else { - stats.addCacheMiss(chars.length()); + STATS.addCacheMiss(chars.length()); return chars.toString(); } }); @@ -61,13 +64,15 @@ public final class StringPool { } } if (COLLECT_STATS) { - stats.addPass(c); + STATS.addPass(c); } return c.toString(); } public static void printStats() { - stats.print(); + if (COLLECT_STATS) { + STATS.print(); + } } private static class Stats { @@ -84,15 +89,15 @@ public final class StringPool { System.err.println("String pool stats"); System.err.println("================="); System.err.println("Hits: " + hits + " (" + (hits * 100 / (total() + 1)) + "% of " + total() + ")"); - System.err.println("Hit length (net savings): " + hitLen + " " + toSize(hitLen)); - System.err.println( - "Total pool size: " + poolContents.getCount() + " strings (" + toSize(poolContents.getSum()) + ")"); + System.err.println("Hit length (net savings): " + hitLen + " (" + toSize(hitLen) + ")"); + System.err.println("Total pool size: " + poolContents.getCount() + " strings (" + toSize(poolContents.getSum()) + ")"); System.err.println("Avg pooled string length: " + poolContents.getAverage() + " chars"); System.err.println("Unpooled string length: " + notPooledAlloc + " chars (" + toSize(notPooledAlloc) + ")"); } private String toSize(long charLen) { - // assuming all pooled chars are latin-1, so 1B in a compressed string (byte[], not char[]) + // Assuming all pooled chars are latin-1, so 1B in a compressed string (byte[], not char[]) + // Before Java 9, it's twice as much if (charLen > (1 << 20)) { return (charLen >> 20) + " MB"; } else if (charLen > (1 << 10)) { From b8af0dda6ea27f55de1c2f6d33f66a06d96d4d1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 29 Apr 2020 00:48:57 +0200 Subject: [PATCH 032/136] Move AbstractTokenManager --- javacc-wrapper.xml | 2 +- .../pmd/lang/ast/{ => impl/javacc}/AbstractTokenManager.java | 5 +---- .../sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java | 5 ++--- pmd-plsql/src/main/ant/alljavacc.xml | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/{ => impl/javacc}/AbstractTokenManager.java (83%) diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index 6addedec11..b303e31e6d 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -59,7 +59,7 @@ - + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/AbstractTokenManager.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractTokenManager.java similarity index 83% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/AbstractTokenManager.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractTokenManager.java index 8bb6bf0469..9635ccc94e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/AbstractTokenManager.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractTokenManager.java @@ -2,19 +2,16 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast; +package net.sourceforge.pmd.lang.ast.impl.javacc; import java.util.HashMap; import java.util.Map; import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; /** * A base class for the token managers generated by JavaCC. - * - * TODO move to impl.javacc package */ public abstract class AbstractTokenManager implements TokenManager { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index 64421a3bbf..69677dd963 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -11,7 +11,7 @@ import java.io.IOException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; -import net.sourceforge.pmd.util.document.TextDocument; +import net.sourceforge.pmd.util.document.TextRegion; /** * PMD flavour of character streams used by JavaCC parsers. @@ -130,8 +130,7 @@ public final class CharStream { private FileLocation endLocation() { - TextDocument textDoc = tokenDoc.getTextDocument(); - return textDoc.toLocation(textDoc.createRegion(getEndOffset(), 0)); + return tokenDoc.getTextDocument().toLocation(TextRegion.fromOffsetLength(getEndOffset(), 0)); } diff --git a/pmd-plsql/src/main/ant/alljavacc.xml b/pmd-plsql/src/main/ant/alljavacc.xml index 180c2bbedf..e33aef3418 100644 --- a/pmd-plsql/src/main/ant/alljavacc.xml +++ b/pmd-plsql/src/main/ant/alljavacc.xml @@ -82,7 +82,7 @@ value="PLSQLNode" /> + value="class PLSQLParserTokenManager extends net.sourceforge.pmd.lang.ast.impl.javacc.AbstractTokenManager" /> public class Date: Tue, 16 Jun 2020 02:35:38 +0200 Subject: [PATCH 033/136] Fix rebase --- .../pmd/cpd/internal/JavaCCTokenizer.java | 4 ++-- .../ast/impl/javacc/io/EscapeAwareReader.java | 21 ++++++++----------- .../impl/javacc/io/FragmentedDocBuilder.java | 3 +-- 3 files changed, 12 insertions(+), 16 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index d39486f6f4..5905745d51 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -18,13 +18,13 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.io.TextFile; +import net.sourceforge.pmd.util.document.io.PmdFiles; public abstract class JavaCCTokenizer implements Tokenizer { @SuppressWarnings("PMD.CloseResource") protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { - TextDocument textDocument = TextDocument.create(TextFile.cpdCompat(sourceCode)); + TextDocument textDocument = TextDocument.create(PmdFiles.cpdCompat(sourceCode)); JavaccTokenDocument tokenDoc = newTokenDoc(textDocument); return makeLexerImpl(CharStream.create(tokenDoc)); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index 9018931592..7866a00f05 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -19,11 +19,10 @@ import net.sourceforge.pmd.util.document.Chars; /** * A reader that may interpret escapes in its input text. It records * where escapes occurred, and can translate an offset in the translated - * document (the "output") to a line/column/offset coordinates in the - * original input. It uses a single char buffer to store both input and - * translated output, and is overall very optimised for the case where - * there are very few escapes. {@link CharStream} is the API to navigate - * on a translated document (with arbitrary backtrack abilities). + * document (the "output") to an offset in the original input. + * The implementation is optimised for the case where there are few escapes. + * {@link CharStream} is the API to navigate on a translated document + * (with arbitrary backtrack abilities). * *

This is useful to back a {@link CharStream} for JavaCC implementation, * but can also be used as a plain {@link Reader} if using other parser/lexer @@ -58,8 +57,7 @@ public class EscapeAwareReader extends Reader { } /** - * Translate all the input (in-place) in the buffer. This is fed to a - * cursor initialized to zero. + * Translate all the input in the buffer. This is fed to a cursor initialized to zero. */ FragmentedDocCursor translate() throws IOException { readUnchecked(null, 0, Integer.MAX_VALUE); @@ -123,11 +121,10 @@ public class EscapeAwareReader extends Reader { } /** - * Returns the max offset, EXclusive, with which we can cut the input - * array from the bufpos to dump it into the output array. This must - * set the {@link #bufpos} to where we should start reading next (INclusive). - * If applicable, it must also replace in the buffer the start of - * the escape with its translation. + * Returns the max offset, EXclusive, up to which we can cut the input + * array from the bufpos to dump it into the output array. + * + * @param maxOff Max offset up to which to read ahead */ protected int gobbleMaxWithoutEscape(int maxOff) throws IOException { return this.bufpos = maxOff; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 8fdb44cb67..2783a07b24 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -30,8 +30,7 @@ final class FragmentedDocBuilder { * This may be empty. */ void recordDelta(int startInInput, int endInInput, Chars translation) { - assert curOffInInput <= startInInput - : "Already moved past " + curOffInInput + ", cannot add delta at " + startInInput; + assert curOffInInput <= startInInput : "Already moved past " + curOffInInput + ", cannot add delta at " + startInInput; assert startInInput <= endInInput : "Offsets must be ordered"; assert translation != null : "Translation cannot be null"; From 9843ac0b9d5fbab5a53efe17b6083779cf3e4023 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 31 Aug 2020 07:58:53 +0200 Subject: [PATCH 034/136] Fix c++ --- .../impl/javacc/io/FragmentedDocCursor.java | 2 +- .../token/internal/BaseTokenFilterTest.java | 5 +++ .../impl/javacc/io/CharStreamImplTest.java | 11 +++-- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 41 ++++++++++++------- .../pmd/lang/cpp/ast/CppEscapeReader.java | 6 +-- .../pmd/cpd/CppCharStreamTest.java | 13 +++--- 6 files changed, 50 insertions(+), 28 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 500f01460c..31092cdadc 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -215,7 +215,7 @@ final class FragmentedDocCursor { @Override public String toString() { - return "Fragment[" + inStart + ".." + outStart + "]\n" + chars; + return "Fragment[" + inStart + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; } } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java index 32dc6c62e3..53eb394d49 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java @@ -41,6 +41,11 @@ public class BaseTokenFilterTest { return null; } + @Override + public TextRegion getRegion() { + return TextRegion.fromBothOffsets(0, text.length()); + } + @Override public boolean isEof() { return text == null; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index fe336ffddb..85f7d32265 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -14,6 +14,8 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; +import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -22,6 +24,7 @@ public class CharStreamImplTest { @Rule public ExpectedException expect = ExpectedException.none(); + private LanguageVersion dummyVersion = LanguageRegistry.getDefaultLanguage().getDefaultVersion(); @Test public void testReadZeroChars() throws IOException { @@ -135,12 +138,12 @@ public class CharStreamImplTest { stream.readChar(); } - public static CharStream simpleCharStream(String abcd) throws IOException { - return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd))); + public CharStream simpleCharStream(String abcd) throws IOException { + return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion))); } - public static CharStream javaCharStream(String abcd) throws IOException { - return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd)) { + public CharStream javaCharStream(String abcd) throws IOException { + return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion)) { @Override public EscapeAwareReader newReader(Chars text) { return new JavaEscapeReader(text); diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 7a15244762..43add8f056 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -4,13 +4,11 @@ package net.sourceforge.pmd.cpd; -import java.io.BufferedReader; import java.io.IOException; import java.util.Properties; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.PMD; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; @@ -21,6 +19,7 @@ import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; +import net.sourceforge.pmd.util.document.io.TextFileContent; /** * The C++ tokenizer. @@ -63,34 +62,46 @@ public class CPPTokenizer extends JavaCCTokenizer { } } - private Chars maybeSkipBlocks(Chars test) throws IOException { + /** + * @param chars Normalized chars + */ + private CharSequence maybeSkipBlocks(Chars chars) { if (!skipBlocks) { - return test; + return chars; } - try (BufferedReader reader = new BufferedReader(test.newReader())) { - StringBuilder filtered = new StringBuilder(test.length()); - String line; - boolean skip = false; - while ((line = reader.readLine()) != null) { - if (skipBlocksStart.equalsIgnoreCase(line.trim())) { + int i = 0; + int lastLineStart = 0; + boolean skip = false; + StringBuilder filtered = new StringBuilder(chars.length()); + while (i < chars.length()) { + if (chars.charAt(i) == TextFileContent.NORMALIZED_LINE_TERM_CHAR) { + Chars lastLine = chars.subSequence(lastLineStart, i); + Chars trimmed = lastLine.trim(); + if (trimmed.contentEquals(skipBlocksStart, true)) { skip = true; - } else if (skip && skipBlocksEnd.equalsIgnoreCase(line.trim())) { + } else if (trimmed.contentEquals(skipBlocksEnd, true)) { skip = false; } if (!skip) { - filtered.append(line); + lastLine.appendChars(filtered); } - // always add a new line to keep the line-numbering - filtered.append(PMD.EOL); + // always add newline, to preserve line numbers + filtered.append(TextFileContent.NORMALIZED_LINE_TERM_CHAR); + lastLineStart = i + 1; } - return Chars.wrap(filtered, false); + i++; } + if (lastLineStart < i && !skip) { + chars.appendChars(filtered, lastLineStart, i - lastLineStart); + } + return filtered; } @Override protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { + textDoc = TextDocument.readOnlyString(maybeSkipBlocks(textDoc.getText()), textDoc.getDisplayName(), textDoc.getLanguageVersion()); return new JavaccTokenDocument(textDoc) { @Override public EscapeAwareReader newReader(Chars text) { diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java index b041556f53..86f247604d 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java @@ -18,13 +18,13 @@ public class CppEscapeReader extends BackslashEscapeReader { @Override protected int handleBackslash(int maxOff, final int backSlashOff) { - int off = backSlashOff; + int off = backSlashOff + 1; if (input.charAt(off) == NEWLINE) { - return recordEscape(backSlashOff, 2, Chars.EMPTY); + return recordEscape(backSlashOff, off + 1, Chars.EMPTY); } else if (input.charAt(off) == CARRIAGE_RETURN) { if (input.charAt(++off) == NEWLINE) { - return recordEscape(backSlashOff, 3, Chars.EMPTY); + return recordEscape(backSlashOff, off + 1, Chars.EMPTY); } } diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 1942bbc2a9..1743e05518 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -13,12 +13,15 @@ import org.junit.Test; import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; +import net.sourceforge.pmd.util.document.io.PmdFiles; +import net.sourceforge.pmd.util.document.io.TextFile; public class CppCharStreamTest { @NonNull public CharStream charStreamFor(String source) throws IOException { - return CharStream.create(new CPPTokenizer().newTokenDoc(TextDocument.readOnlyString(source))); + TextDocument textDoc = TextDocument.create(PmdFiles.forString(source, TextFile.UNKNOWN_FILENAME, PmdFiles.dummyCpdVersion())); + return CharStream.create(new CPPTokenizer().newTokenDoc(textDoc)); } @Test @@ -37,7 +40,7 @@ public class CppCharStreamTest { @Test public void testBackup() throws IOException { // note that the \r is normalized to a \n by the TextFile - CharStream stream = charStreamFor("a\\b\\\rc"); + CharStream stream = charStreamFor("a\\b\\qc"); assertStream(stream, "a\\b\\qc"); } @@ -49,8 +52,8 @@ public class CppCharStreamTest { assertEquals(token + " char at " + i + ": " + token.charAt(i) + " != " + c, token.charAt(i), c); } assertEquals(token, stream.getTokenImage()); - StringBuilder sb = new StringBuilder(); - stream.appendSuffix(sb, token.length()); - assertEquals(token, sb.toString()); + // StringBuilder sb = new StringBuilder(); + // stream.appendSuffix(sb, token.length()); + // assertEquals(token, sb.toString()); } } From 3481e6b699b8bad3187809751c6c101f87512183 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 31 Aug 2020 19:12:39 +0200 Subject: [PATCH 035/136] Checkstyle --- .../main/java/net/sourceforge/pmd/PMD.java | 4 ---- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 2 -- .../pmd/lang/ast/impl/javacc/StringPool.java | 21 ++++++++++--------- .../impl/javacc/io/BackslashEscapeReader.java | 10 ++++++--- .../ast/impl/javacc/io/EscapeAwareReader.java | 3 ++- .../impl/javacc/io/FragmentedDocCursor.java | 4 +++- .../ast/impl/javacc/io/JavaEscapeReader.java | 2 +- .../impl/javacc/io/CharStreamImplTest.java | 1 + .../impl/javacc/io/JavaEscapeReaderTest.java | 6 ++---- 9 files changed, 27 insertions(+), 26 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java index ec7476f730..e24bdbfe5c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/PMD.java @@ -32,10 +32,6 @@ import net.sourceforge.pmd.cli.PMDParameters; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.lang.LanguageVersionDiscoverer; -import net.sourceforge.pmd.lang.LanguageVersionHandler; -import net.sourceforge.pmd.lang.Parser; -import net.sourceforge.pmd.lang.ParserOptions; -import net.sourceforge.pmd.lang.ast.impl.javacc.StringPool; import net.sourceforge.pmd.processor.AbstractPMDProcessor; import net.sourceforge.pmd.renderers.Renderer; import net.sourceforge.pmd.reporting.GlobalAnalysisListener; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index 866ba74f3f..382484a6d0 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -4,8 +4,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import org.apache.commons.lang3.StringUtils; - import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.FileLocation; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java index a8a238f473..ace5c6a188 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java @@ -49,16 +49,17 @@ public final class StringPool { if (doPool && c instanceof Chars) { if (COLLECT_STATS) { - return pool.compute((Chars) c, - (chars, s) -> { - if (s != null) { - STATS.addCacheHit(s.length()); - return s; - } else { - STATS.addCacheMiss(chars.length()); - return chars.toString(); - } - }); + return pool.compute( + (Chars) c, + (chars, s) -> { + if (s != null) { + STATS.addCacheHit(s.length()); + return s; + } else { + STATS.addCacheMiss(chars.length()); + return chars.toString(); + } + }); } else { return pool.computeIfAbsent((Chars) c, Chars::toString); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java index 54724841a7..12cb6e2ad4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java @@ -33,13 +33,17 @@ public abstract class BackslashEscapeReader extends EscapeAwareReader { @Override protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { int off = this.bufpos; - boolean noBackSlash = false; + boolean seenBackslash = true; int notEscapeEnd = this.savedNotEscapeSpecialEnd; - while (off < maxOff && (noBackSlash = input.charAt(off) != BACKSLASH || notEscapeEnd < off)) { + while (off < maxOff) { + seenBackslash = input.charAt(off) == BACKSLASH && notEscapeEnd >= off; + if (seenBackslash) { + break; + } off++; } - if (noBackSlash || off == maxOff) { + if (!seenBackslash || off == maxOff) { this.bufpos = off; return off; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index 7866a00f05..da50d0dade 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -127,7 +127,8 @@ public class EscapeAwareReader extends Reader { * @param maxOff Max offset up to which to read ahead */ protected int gobbleMaxWithoutEscape(int maxOff) throws IOException { - return this.bufpos = maxOff; + this.bufpos = maxOff; + return maxOff; } protected int recordEscape(final int startOffsetInclusive, int endOffsetExclusive, Chars translation) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 31092cdadc..51991513cf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -11,6 +11,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.util.document.Chars; +@SuppressWarnings("PMD.CompareObjectsWithEquals") final class FragmentedDocCursor { private static final EOFException EOF = new EOFException(); @@ -173,7 +174,8 @@ final class FragmentedDocCursor { this.outStart = prev.outEnd(); this.inStart = prev.inEnd(); } else { - this.outStart = this.inStart = 0; + this.inStart = 0; + this.outStart = 0; } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index 3ae12bb84e..dc04bd7a60 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -46,7 +46,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { private Chars escapeValue(int posOfFirstBackSlash, int offOfTheU) throws IOException { try { char c = (char) - ( hexVal(input.charAt(++offOfTheU)) << 12 + ( hexVal(input.charAt(++offOfTheU)) << 12 // SUPPRESS CHECKSTYLE paren pad | hexVal(input.charAt(++offOfTheU)) << 8 | hexVal(input.charAt(++offOfTheU)) << 4 | hexVal(input.charAt(++offOfTheU)) diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index 85f7d32265..1803f3e419 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -164,6 +164,7 @@ public class CharStreamImplTest { expect.expect(IllegalArgumentException.class); stream.backup(10); } + @Test public void testBacktrackTooMuch2() throws IOException { diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index 9443514679..28a90609dc 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -67,7 +67,7 @@ public class JavaEscapeReaderTest { } @Test - public void testNotAnEscape1Read3_SplitInTheMiddleOfBackslashes() throws IOException { + public void testNotAnEscape1Read3SplitInTheMiddleOfBackslashes() throws IOException { String input = "abc\\\\\\dede"; // ^ @@ -158,6 +158,4 @@ public class JavaEscapeReaderTest { contents.getChars(0, contents.length(), chars2, off); Assert.assertArrayEquals(chars2, chars); } - - -} \ No newline at end of file +} From 9a4fc3cee6977edfaefb1bd7bed556b18f481165 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Tue, 1 Sep 2020 23:38:43 +0200 Subject: [PATCH 036/136] Move factory methods to the interface --- .../pmd/cpd/internal/AntlrTokenizer.java | 2 +- .../pmd/util/document/CpdCompat.java | 53 ------------------- .../pmd/util/document/TextDocument.java | 8 --- .../pmd/util/treeexport/TreeExportCli.java | 2 +- 4 files changed, 2 insertions(+), 63 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java index f0659b952b..81ec427a09 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java @@ -19,7 +19,7 @@ import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.CpdCompat; +import net.sourceforge.pmd.util.document.io.CpdCompat; /** * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java deleted file mode 100644 index fa677a573f..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.util.document; - -import net.sourceforge.pmd.cpd.SourceCode; -import net.sourceforge.pmd.lang.BaseLanguageModule; -import net.sourceforge.pmd.lang.Language; -import net.sourceforge.pmd.lang.LanguageVersion; - -/** - * Compatibility APIs, to be removed before PMD 7 is out. - */ -@Deprecated -public final class CpdCompat { - - private CpdCompat() { - // utility class - } - - - /** The language version must be non-null. */ - @Deprecated - private static final Language DUMMY_LANG = new BaseLanguageModule("dummy", "dummy", "dummy", "dummy") { - { - addDefaultVersion("", parserOptions -> task -> { - throw new UnsupportedOperationException(); - }); - } - - }; - - @Deprecated - public static LanguageVersion dummyVersion() { - return DUMMY_LANG.getDefaultVersion(); - } - - /** - * Bridges {@link SourceCode} with {@link TextFile}. This allows - * javacc tokenizers to work on text documents. - * - * @deprecated This is only a transitional API for the PMD 7 branch - */ - @Deprecated - public static TextFile cpdCompat(SourceCode sourceCode) { - return TextFile.forCharSeq( - sourceCode.getCodeBuffer(), - sourceCode.getFileName(), - dummyVersion() - ); - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 3f38ed2d31..41ab4a550f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.util.document; import java.io.Closeable; import java.io.IOException; -import java.io.Reader; import org.checkerframework.checker.nullness.qual.NonNull; @@ -70,13 +69,6 @@ public interface TextDocument extends Closeable { */ TextFileContent getContent(); - /** - * Returns a reader over the text of this document. - */ - default Reader newReader() { - return getText().newReader(); - } - /** * Returns the length in characters of the {@linkplain #getText() text}. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java index 16455a62aa..a84bb49b7e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java @@ -168,7 +168,7 @@ public class TreeExportCli { throw bail("One of --file or --read-stdin must be mentioned"); } else if (readStdin) { System.err.println("Reading from stdin..."); - textFile = TextFile.forCharSeq(readFromSystemIn(), "stdin", langVersion); + textFile = TextFile.forCharSeq(readFromSystemIn(), "stdin", langVersion).build(); } else { textFile = TextFile.forPath(Paths.get(file), Charset.forName(encoding), langVersion).build(); } From eaed142496c26ed10ca5286ce7aa509fc8ddf03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 02:00:41 +0200 Subject: [PATCH 037/136] Move text file stuff into the main document package --- .../pmd/cpd/internal/AntlrTokenizer.java | 2 +- .../pmd/cpd/internal/JavaCCTokenizer.java | 4 +- .../pmd/util/document/CpdCompat.java | 53 +++++++++++++++++++ .../pmd/util/treeexport/TreeExportCli.java | 2 +- .../token/internal/BaseTokenFilterTest.java | 5 -- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 2 +- .../pmd/cpd/CppCharStreamTest.java | 6 +-- 7 files changed, 61 insertions(+), 13 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java index 81ec427a09..f0659b952b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java @@ -19,7 +19,7 @@ import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.io.CpdCompat; +import net.sourceforge.pmd.util.document.CpdCompat; /** * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 5905745d51..a068485618 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -17,14 +17,14 @@ import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.io.PmdFiles; public abstract class JavaCCTokenizer implements Tokenizer { @SuppressWarnings("PMD.CloseResource") protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { - TextDocument textDocument = TextDocument.create(PmdFiles.cpdCompat(sourceCode)); + TextDocument textDocument = TextDocument.create(CpdCompat.cpdCompat(sourceCode)); JavaccTokenDocument tokenDoc = newTokenDoc(textDocument); return makeLexerImpl(CharStream.create(tokenDoc)); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java new file mode 100644 index 0000000000..fa677a573f --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/CpdCompat.java @@ -0,0 +1,53 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.document; + +import net.sourceforge.pmd.cpd.SourceCode; +import net.sourceforge.pmd.lang.BaseLanguageModule; +import net.sourceforge.pmd.lang.Language; +import net.sourceforge.pmd.lang.LanguageVersion; + +/** + * Compatibility APIs, to be removed before PMD 7 is out. + */ +@Deprecated +public final class CpdCompat { + + private CpdCompat() { + // utility class + } + + + /** The language version must be non-null. */ + @Deprecated + private static final Language DUMMY_LANG = new BaseLanguageModule("dummy", "dummy", "dummy", "dummy") { + { + addDefaultVersion("", parserOptions -> task -> { + throw new UnsupportedOperationException(); + }); + } + + }; + + @Deprecated + public static LanguageVersion dummyVersion() { + return DUMMY_LANG.getDefaultVersion(); + } + + /** + * Bridges {@link SourceCode} with {@link TextFile}. This allows + * javacc tokenizers to work on text documents. + * + * @deprecated This is only a transitional API for the PMD 7 branch + */ + @Deprecated + public static TextFile cpdCompat(SourceCode sourceCode) { + return TextFile.forCharSeq( + sourceCode.getCodeBuffer(), + sourceCode.getFileName(), + dummyVersion() + ); + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java index a84bb49b7e..16455a62aa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/treeexport/TreeExportCli.java @@ -168,7 +168,7 @@ public class TreeExportCli { throw bail("One of --file or --read-stdin must be mentioned"); } else if (readStdin) { System.err.println("Reading from stdin..."); - textFile = TextFile.forCharSeq(readFromSystemIn(), "stdin", langVersion).build(); + textFile = TextFile.forCharSeq(readFromSystemIn(), "stdin", langVersion); } else { textFile = TextFile.forPath(Paths.get(file), Charset.forName(encoding), langVersion).build(); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java index 53eb394d49..16bcdb6b01 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java @@ -56,11 +56,6 @@ public class BaseTokenFilterTest { return text; } - @Override - public TextRegion getRegion() { - return TextRegion.fromBothOffsets(0, text.length()); - } - @Override public FileLocation getReportLocation() { return FileLocation.location("n/a", 0, 0, 0, 0); diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 43add8f056..dc764c9939 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -19,7 +19,7 @@ import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.io.TextFileContent; +import net.sourceforge.pmd.util.document.TextFileContent; /** * The C++ tokenizer. diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 1743e05518..42eb5c1dd9 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -12,15 +12,15 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.io.PmdFiles; -import net.sourceforge.pmd.util.document.io.TextFile; +import net.sourceforge.pmd.util.document.TextFile; public class CppCharStreamTest { @NonNull public CharStream charStreamFor(String source) throws IOException { - TextDocument textDoc = TextDocument.create(PmdFiles.forString(source, TextFile.UNKNOWN_FILENAME, PmdFiles.dummyCpdVersion())); + TextDocument textDoc = TextDocument.readOnlyString(source, TextFile.UNKNOWN_FILENAME, CpdCompat.dummyVersion()); return CharStream.create(new CPPTokenizer().newTokenDoc(textDoc)); } From a6e1f634f58bffc364a68c3e7af70c1dc0411438 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 03:07:18 +0200 Subject: [PATCH 038/136] Add MalformedSourceException --- .../pmd/cpd/internal/JavaCCTokenizer.java | 9 +++--- .../pmd/lang/ast/FileAnalysisException.java | 2 +- .../pmd/lang/ast/GenericToken.java | 7 ++++- .../impl/javacc/io/BackslashEscapeReader.java | 6 ++-- .../lang/ast/impl/javacc/io/CharStream.java | 2 +- .../ast/impl/javacc/io/EscapeAwareReader.java | 8 ++--- .../ast/impl/javacc/io/JavaEscapeReader.java | 12 ++------ .../javacc/io/MalformedSourceException.java | 29 +++++++++++++++++++ 8 files changed, 51 insertions(+), 24 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index a068485618..9e54955e00 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -13,17 +13,18 @@ import net.sourceforge.pmd.cpd.Tokens; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.ast.FileAnalysisException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; public abstract class JavaCCTokenizer implements Tokenizer { @SuppressWarnings("PMD.CloseResource") - protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException { + protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException, MalformedSourceException { TextDocument textDocument = TextDocument.create(CpdCompat.cpdCompat(sourceCode)); JavaccTokenDocument tokenDoc = newTokenDoc(textDocument); return makeLexerImpl(CharStream.create(tokenDoc)); @@ -49,15 +50,15 @@ public abstract class JavaCCTokenizer implements Tokenizer { @Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) throws IOException { - TokenManager tokenManager = getLexerForSource(sourceCode); try { + TokenManager tokenManager = getLexerForSource(sourceCode); final TokenFilter tokenFilter = getTokenFilter(tokenManager); JavaccToken currentToken = tokenFilter.getNextToken(); while (currentToken != null) { tokenEntries.add(processToken(tokenEntries, currentToken, sourceCode.getFileName())); currentToken = tokenFilter.getNextToken(); } - } catch (TokenMgrError e) { + } catch (FileAnalysisException e) { throw e.setFileName(sourceCode.getFileName()); } finally { tokenEntries.add(TokenEntry.getEOF()); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java index a60a8cb54d..dbe7fa4135 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java @@ -39,7 +39,7 @@ public class FileAnalysisException extends RuntimeException { super(message, cause); } - FileAnalysisException setFileName(String filename) { + public FileAnalysisException setFileName(String filename) { this.filename = Objects.requireNonNull(filename); return this; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java index f4886fcfa6..23063a584c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/GenericToken.java @@ -9,6 +9,7 @@ import java.util.Iterator; import org.apache.commons.lang3.StringUtils; import net.sourceforge.pmd.internal.util.IteratorUtil; +import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.Reportable; import net.sourceforge.pmd.util.document.TextRegion; @@ -56,7 +57,11 @@ public interface GenericToken> extends Comparable, * @param charSeq A character sequence */ default boolean imageEquals(CharSequence charSeq) { - return StringUtils.equals(getImageCs(), charSeq); + CharSequence imageCs = getImageCs(); + if (imageCs instanceof Chars) { + return ((Chars) imageCs).contentEquals(charSeq); + } + return StringUtils.equals(imageCs, charSeq); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java index 12cb6e2ad4..9bed9435d9 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java @@ -6,8 +6,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; -import java.io.IOException; - import net.sourceforge.pmd.util.document.Chars; /** @@ -31,7 +29,7 @@ public abstract class BackslashEscapeReader extends EscapeAwareReader { } @Override - protected int gobbleMaxWithoutEscape(final int maxOff) throws IOException { + protected int gobbleMaxWithoutEscape(final int maxOff) throws MalformedSourceException { int off = this.bufpos; boolean seenBackslash = true; int notEscapeEnd = this.savedNotEscapeSpecialEnd; @@ -51,7 +49,7 @@ public abstract class BackslashEscapeReader extends EscapeAwareReader { return handleBackslash(maxOff, off); } - protected abstract int handleBackslash(int maxOff, int firstBackslashOff) throws IOException; + protected abstract int handleBackslash(int maxOff, int firstBackslashOff) throws MalformedSourceException; @Override protected int recordEscape(int startOffsetInclusive, int endOffsetExclusive, Chars translation) { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index 69677dd963..5b2fd779bf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -31,7 +31,7 @@ public final class CharStream { /** * Create a new char stream for the given document. */ - public static CharStream create(JavaccTokenDocument doc) throws IOException { + public static CharStream create(JavaccTokenDocument doc) throws IOException, MalformedSourceException { try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { return new CharStream(doc, reader.translate()); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index da50d0dade..53a22554c4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -59,14 +59,14 @@ public class EscapeAwareReader extends Reader { /** * Translate all the input in the buffer. This is fed to a cursor initialized to zero. */ - FragmentedDocCursor translate() throws IOException { + FragmentedDocCursor translate() throws IOException, MalformedSourceException { readUnchecked(null, 0, Integer.MAX_VALUE); return escapes.newCursor(); } @Override - public int read(final char[] cbuf, final int off, int len) throws IOException { + public int read(final char[] cbuf, final int off, int len) throws IOException, MalformedSourceException { if (off < 0 || len < 0 || len + off > cbuf.length) { throw new IndexOutOfBoundsException("cbuf len=" + cbuf.length + " off=" + off + " len=" + len); } @@ -74,7 +74,7 @@ public class EscapeAwareReader extends Reader { } // if cbuf is null we just want to record escapes - private int readUnchecked(char @Nullable [] cbuf, int off, int len) throws IOException { + private int readUnchecked(char @Nullable [] cbuf, int off, int len) throws IOException, MalformedSourceException { ensureOpen(); if (this.bufpos == input.length()) { return -1; @@ -126,7 +126,7 @@ public class EscapeAwareReader extends Reader { * * @param maxOff Max offset up to which to read ahead */ - protected int gobbleMaxWithoutEscape(int maxOff) throws IOException { + protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException { this.bufpos = maxOff; return maxOff; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index dc04bd7a60..d404851af4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -4,8 +4,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; -import java.io.IOException; - import net.sourceforge.pmd.util.document.Chars; /** @@ -20,7 +18,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { } @Override - protected int handleBackslash(final int maxOff, final int firstBackslashOff) throws IOException { + protected int handleBackslash(final int maxOff, final int firstBackslashOff) throws MalformedSourceException { int off = firstBackslashOff; while (off < input.length() && input.charAt(off) == '\\') { off++; @@ -43,7 +41,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { } } - private Chars escapeValue(int posOfFirstBackSlash, int offOfTheU) throws IOException { + private Chars escapeValue(int posOfFirstBackSlash, int offOfTheU) throws MalformedSourceException { try { char c = (char) ( hexVal(input.charAt(++offOfTheU)) << 12 // SUPPRESS CHECKSTYLE paren pad @@ -54,11 +52,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { return Chars.wrap(Character.toString(c)); } catch (NumberFormatException | IndexOutOfBoundsException e) { - String message = "Invalid escape sequence at line " - + getLine(posOfFirstBackSlash) + ", column " - + getColumn(posOfFirstBackSlash); - - throw new IOException(message, e); + throw new MalformedSourceException("Invalid escape sequence", e, posOfFirstBackSlash, getLine(posOfFirstBackSlash), getColumn(posOfFirstBackSlash)); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java new file mode 100644 index 0000000000..454b06f054 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java @@ -0,0 +1,29 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.ast.impl.javacc.io; + +import net.sourceforge.pmd.lang.ast.FileAnalysisException; +import net.sourceforge.pmd.lang.ast.TokenMgrError; + +/** + * + */ +public class MalformedSourceException extends FileAnalysisException { + + private final int offset; + private final int line; + private final int col; + + public MalformedSourceException(String message, Throwable cause, int offset, int line, int col) { + super(message, cause); + this.offset = offset; + this.line = line; + this.col = col; + } + + public TokenMgrError toLexException(String filename) { + return new TokenMgrError(line, col, filename, getMessage(), getCause()); + } +} From 2d9d74e1ec2be0baeebea6b2dfb84628d0ec076c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 03:33:56 +0200 Subject: [PATCH 039/136] Cleanup exception messages --- .../pmd/lang/apex/ast/ApexParser.java | 6 ++---- .../pmd/lang/ast/FileAnalysisException.java | 17 +++++++++++++++++ .../pmd/lang/ast/ParseException.java | 10 +++++----- .../pmd/lang/ast/TokenMgrError.java | 10 ++++++---- .../ast/impl/javacc/io/JavaEscapeReader.java | 2 +- .../javacc/io/MalformedSourceException.java | 18 +++++++++++------- .../pmd/lang/java/ast/ParserCornersTest.java | 6 +++--- 7 files changed, 45 insertions(+), 24 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexParser.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexParser.java index b72a761991..511df40ecd 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexParser.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexParser.java @@ -27,9 +27,7 @@ public final class ApexParser implements Parser { final Compilation astRoot = CompilerService.INSTANCE.parseApex(task.getTextDocument()); - if (astRoot == null) { - throw new ParseException("Couldn't parse the source - there is not root node - Syntax Error??"); - } + assert astRoot != null : "Normally replaced by Compilation.INVALID"; final ApexTreeBuilder treeBuilder = new ApexTreeBuilder(task.getTextDocument(), task.getCommentMarker()); AbstractApexNode treeRoot = treeBuilder.build(astRoot); @@ -37,7 +35,7 @@ public final class ApexParser implements Parser { fileNode.setNoPmdComments(treeBuilder.getSuppressMap()); return fileNode; } catch (apex.jorje.services.exception.ParseException e) { - throw new ParseException(e); + throw new ParseException(e).setFileName(task.getFileDisplayName()); } } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java index dbe7fa4135..d3a7d5e2a2 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/FileAnalysisException.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.lang.ast; import java.util.Objects; +import org.apache.commons.lang3.StringUtils; import org.checkerframework.checker.nullness.qual.NonNull; import net.sourceforge.pmd.util.document.TextFile; @@ -55,6 +56,22 @@ public class FileAnalysisException extends RuntimeException { return filename; } + @Override + public String getMessage() { + return errorKind() + StringUtils.uncapitalize(positionToString()) + ": " + super.getMessage(); + } + + protected String errorKind() { + return "Error"; + } + + protected String positionToString() { + if (hasFileName()) { + return " in file '" + getFileName() + "'"; + } + return ""; + } + /** * Wraps the cause into an analysis exception. If it is itself an analysis diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/ParseException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/ParseException.java index bc347e1cd4..27598f83e5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/ParseException.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/ParseException.java @@ -40,11 +40,6 @@ public class ParseException extends FileAnalysisException { this.currentToken = null; } - public ParseException(String message, Throwable cause) { - super(message, cause); - this.currentToken = null; - } - public ParseException(String message, JavaccToken token) { super(message); this.currentToken = token; @@ -59,6 +54,11 @@ public class ParseException extends FileAnalysisException { currentToken = currentTokenVal; } + @Override + protected String errorKind() { + return "Parse exception"; + } + /** * It uses "currentToken" and "expectedTokenSequences" to generate a parse * error message and returns it. If this object has been created diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java index 3625251bbe..06dd3b01a1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TokenMgrError.java @@ -53,12 +53,14 @@ public final class TokenMgrError extends FileAnalysisException { return column; } - + @Override + protected String positionToString() { + return super.positionToString() + " at line " + line + ", column " + column; + } @Override - public String getMessage() { - String leader = hasFileName() ? "Lexical error in file " + getFileName() : "Lexical error"; - return leader + " at line " + line + ", column " + column + ". Encountered: " + super.getMessage(); + protected String errorKind() { + return "Lexical error"; } /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java index d404851af4..7ccea7bf0c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java @@ -52,7 +52,7 @@ public final class JavaEscapeReader extends BackslashEscapeReader { return Chars.wrap(Character.toString(c)); } catch (NumberFormatException | IndexOutOfBoundsException e) { - throw new MalformedSourceException("Invalid escape sequence", e, posOfFirstBackSlash, getLine(posOfFirstBackSlash), getColumn(posOfFirstBackSlash)); + throw new MalformedSourceException("Invalid unicode escape " + " at line", e, getLine(posOfFirstBackSlash), getColumn(posOfFirstBackSlash)); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java index 454b06f054..b8210adb1c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java @@ -5,25 +5,29 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import net.sourceforge.pmd.lang.ast.FileAnalysisException; -import net.sourceforge.pmd.lang.ast.TokenMgrError; /** - * + * A {@link FileAnalysisException} thrown when the source format is invalid, + * for example if some unicode escapes cannot be translated. */ public class MalformedSourceException extends FileAnalysisException { - private final int offset; private final int line; private final int col; - public MalformedSourceException(String message, Throwable cause, int offset, int line, int col) { + public MalformedSourceException(String message, Throwable cause, int line, int col) { super(message, cause); - this.offset = offset; this.line = line; this.col = col; } - public TokenMgrError toLexException(String filename) { - return new TokenMgrError(line, col, filename, getMessage(), getCause()); + @Override + protected String positionToString() { + return super.positionToString() + " at line " + line + ", column " + col; + } + + @Override + protected String errorKind() { + return "Source format error"; } } diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java index 2fedde03b3..eced79d5ad 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java @@ -14,7 +14,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; -import net.sourceforge.pmd.lang.ast.TokenMgrError; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.ast.test.BaseParsingHelper; import net.sourceforge.pmd.lang.java.BaseJavaTreeDumpTest; import net.sourceforge.pmd.lang.java.JavaParsingHelper; @@ -38,8 +38,8 @@ public class ParserCornersTest extends BaseJavaTreeDumpTest { @Test public void testInvalidUnicodeEscape() { - expect.expect(TokenMgrError.class); // previously Error - expect.expectMessage("Lexical error at line 1, column 2. Encountered: Invalid unicode escape"); + expect.expect(MalformedSourceException.class); // previously Error + expect.expectMessage("Source format error at line 1, column 1: Invalid unicode escape"); java.parse("\\u00k0"); } From 3fbb8b51d748d5bded9f43aa1c5a96d276fade75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 04:21:47 +0200 Subject: [PATCH 040/136] Cleanup --- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 5 ++++ .../impl/javacc/io/FragmentedDocCursor.java | 23 +++++++------------ .../pmd/util/document/RootTextDocument.java | 5 ---- .../pmd/util/document/TextDocument.java | 4 +++- 4 files changed, 16 insertions(+), 21 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index 382484a6d0..ab9855d46f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -140,6 +140,11 @@ public class JavaccToken implements GenericToken { return image; } + @Override + public String getImage() { + return document.computeImage(this); + } + @Override public TextRegion getRegion() { return TextRegion.fromBothOffsets(startOffset, endOffset); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java index 51991513cf..9f858b337a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java @@ -82,7 +82,7 @@ final class FragmentedDocCursor { if (cur == mark || cur.outStart() <= curOutPos - suffixLen) { // entire suffix is in the last fragment, fast path - cur.chars.appendChars(sb, curOutPos - cur.outStart - suffixLen, suffixLen); + cur.chars.appendChars(sb, curOutPos - cur.outStart() - suffixLen, suffixLen); } else { int suffixStart = curOutPos - suffixLen; Fragment f = findBackwards(suffixStart); @@ -161,8 +161,6 @@ final class FragmentedDocCursor { final @Nullable Fragment prev; @Nullable Fragment next; - private final int outStart; - private final int inStart; private final int inLength; Fragment(@Nullable Fragment prev, int inLength, Chars chars) { @@ -171,24 +169,19 @@ final class FragmentedDocCursor { this.inLength = inLength; if (prev != null) { prev.next = this; - this.outStart = prev.outEnd(); - this.inStart = prev.inEnd(); - } else { - this.inStart = 0; - this.outStart = 0; } } void appendAbs(StringBuilder sb, int absOffset, int absEndOffset) { - chars.appendChars(sb, absOffset - outStart, absEndOffset - absOffset); + chars.appendChars(sb, absOffset - outStart(), absEndOffset - absOffset); } char charAt(int absPos) { - return chars.charAt(absPos - outStart); + return chars.charAt(absPos - outStart()); } int outStart() { - return outStart; + return prev != null ? prev.outEnd() : 0; } int outLen() { @@ -196,11 +189,11 @@ final class FragmentedDocCursor { } int outEnd() { - return outStart + outLen(); + return outStart() + outLen(); } int inStart() { - return inStart; + return prev != null ? prev.inEnd() : 0; } int inLen() { @@ -208,7 +201,7 @@ final class FragmentedDocCursor { } int inEnd() { - return inStart + inLength; + return inStart() + inLen(); } int outToIn(int outOffset) { @@ -217,7 +210,7 @@ final class FragmentedDocCursor { @Override public String toString() { - return "Fragment[" + inStart + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; + return "Fragment[" + inStart() + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; } } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index 52bcba9838..a2db0ea2dd 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -106,11 +106,6 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { return content; } - @Override - public Chars sliceText(TextRegion region) { - return getText().subSequence(region.getStartOffset(), region.getEndOffset()); - } - private static final String NOT_IN_RANGE = "Region [start=%d, end=%d[ is not in range of this document (length %d)"; private static final String INVALID_LINE_RANGE = "Line range %d..%d is not in range of this document (%d lines) (line numbers are 1-based)"; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 41ab4a550f..ef5cf324a7 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -61,7 +61,9 @@ public interface TextDocument extends Closeable { /** * Returns a region of the {@linkplain #getText() text} as a character sequence. */ - Chars sliceText(TextRegion region); + default Chars sliceText(TextRegion region) { + return getText().subSequence(region.getStartOffset(), region.getEndOffset()); + } /** From 51170633b08983ff50b4d0fa5fd84e6b588753ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 05:05:05 +0200 Subject: [PATCH 041/136] Remove fragmented doc cursor --- .../pmd/cache/AbstractAnalysisCache.java | 2 +- .../lang/ast/impl/javacc/io/CharStream.java | 48 ++-- .../ast/impl/javacc/io/EscapeAwareReader.java | 5 +- .../impl/javacc/io/FragmentedDocBuilder.java | 158 ++++++++++++- .../impl/javacc/io/FragmentedDocCursor.java | 216 ------------------ .../pmd/util/document/RootTextDocument.java | 9 +- .../pmd/util/document/TextDocument.java | 12 +- .../pmd/util/document/TextRegion.java | 9 + 8 files changed, 210 insertions(+), 249 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cache/AbstractAnalysisCache.java b/pmd-core/src/main/java/net/sourceforge/pmd/cache/AbstractAnalysisCache.java index 8c6e835fd7..c271d1551b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cache/AbstractAnalysisCache.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cache/AbstractAnalysisCache.java @@ -67,7 +67,7 @@ public abstract class AbstractAnalysisCache implements AnalysisCache { @Override public boolean isUpToDate(final TextDocument document) { // There is a new file being analyzed, prepare entry in updated cache - final AnalysisResult updatedResult = new AnalysisResult(document.getContent().getCheckSum(), new ArrayList<>()); + final AnalysisResult updatedResult = new AnalysisResult(document.getChecksum(), new ArrayList<>()); updatedResultsCache.put(document.getPathId(), updatedResult); // Now check the old cache diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java index 5b2fd779bf..68e3264f6b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java @@ -11,6 +11,7 @@ import java.io.IOException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; +import net.sourceforge.pmd.util.document.TextDocument; import net.sourceforge.pmd.util.document.TextRegion; /** @@ -18,14 +19,20 @@ import net.sourceforge.pmd.util.document.TextRegion; */ public final class CharStream { - private final JavaccTokenDocument tokenDoc; - private final FragmentedDocCursor cursor; - private final boolean useMarkSuffix; + private static final EOFException EOF = new EOFException(); - private CharStream(JavaccTokenDocument tokenDoc, FragmentedDocCursor cursor) { + private final JavaccTokenDocument tokenDoc; + private final TextDocument textDoc; + private final Chars chars; + private final boolean useMarkSuffix; + private int curOffset; + private int markOffset = -1; + + private CharStream(JavaccTokenDocument tokenDoc, TextDocument textDoc) { this.tokenDoc = tokenDoc; - this.cursor = cursor; - useMarkSuffix = tokenDoc.useMarkSuffix(); + this.textDoc = textDoc; + this.chars = textDoc.getText(); + this.useMarkSuffix = tokenDoc.useMarkSuffix(); } /** @@ -33,7 +40,7 @@ public final class CharStream { */ public static CharStream create(JavaccTokenDocument doc) throws IOException, MalformedSourceException { try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { - return new CharStream(doc, reader.translate()); + return new CharStream(doc, reader.translate(doc.getTextDocument())); } } @@ -46,7 +53,10 @@ public final class CharStream { * @throws EOFException Upon EOF */ public char readChar() throws EOFException { - return cursor.next(); + if (curOffset == chars.length()) { + throw EOF; + } + return chars.charAt(curOffset++); } @@ -57,8 +67,8 @@ public final class CharStream { * backup correctly. */ public char markTokenStart() throws EOFException { - cursor.mark(); - return cursor.next(); + markOffset = curOffset; + return readChar(); } @@ -75,7 +85,12 @@ public final class CharStream { * to the current buffer position. */ public Chars getTokenImageCs() { - return cursor.getMarkImage(); + assert markOffset >= 0; + return chars.slice(markOffset, markLen()); + } + + private int markLen() { + return curOffset - markOffset; } @@ -90,7 +105,8 @@ public final class CharStream { */ public void appendSuffix(StringBuilder sb, int len) { if (useMarkSuffix) { - cursor.appendMarkSuffix(sb, len); + assert len <= markLen() : "Suffix is greater than the mark length? " + len + " > " + markLen(); + chars.appendChars(sb, curOffset - len, len); } // otherwise dead code, kept because Javacc's argument expressions do side effects } @@ -108,7 +124,7 @@ public final class CharStream { * number of read chars */ public void backup(int amount) { - cursor.backup(amount); + curOffset -= amount; } /** @@ -130,19 +146,19 @@ public final class CharStream { private FileLocation endLocation() { - return tokenDoc.getTextDocument().toLocation(TextRegion.fromOffsetLength(getEndOffset(), 0)); + return textDoc.toLocation(TextRegion.caretAt(getEndOffset())); } /** Returns the start offset of the current token (in the original source), inclusive. */ public int getStartOffset() { - return cursor.markInOffset(); + return textDoc.translateOffset(markOffset); } /** Returns the end offset of the current token (in the original source), exclusive. */ public int getEndOffset() { - return cursor.curInOffset(); + return textDoc.translateOffset(curOffset); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index 53a22554c4..4a12aa2cae 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -15,6 +15,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; /** * A reader that may interpret escapes in its input text. It records @@ -59,9 +60,9 @@ public class EscapeAwareReader extends Reader { /** * Translate all the input in the buffer. This is fed to a cursor initialized to zero. */ - FragmentedDocCursor translate() throws IOException, MalformedSourceException { + TextDocument translate(TextDocument source) throws IOException, MalformedSourceException { readUnchecked(null, 0, Integer.MAX_VALUE); - return escapes.newCursor(); + return escapes.build(source); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 2783a07b24..006767e3f1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -5,8 +5,16 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.FragmentedDocCursor.Fragment; +import java.io.IOException; + +import org.apache.commons.lang3.NotImplementedException; +import org.checkerframework.checker.nullness.qual.Nullable; + +import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.FileLocation; +import net.sourceforge.pmd.util.document.TextDocument; +import net.sourceforge.pmd.util.document.TextRegion; final class FragmentedDocBuilder { @@ -53,14 +61,12 @@ final class FragmentedDocBuilder { this.curOffInInput = endInInput; } - /** - * Finalize the construction process. - */ - FragmentedDocCursor newCursor() { + TextDocument build(TextDocument document) { if (firstFragment == null) { // No deltas in whole document, there's a single fragment // This is the case for > 97% of Java files (source: OpenJDK) - return new FragmentedDocCursor(new Fragment(null, mainBuf.length(), mainBuf)); + Fragment fragment = new Fragment(null, mainBuf.length(), mainBuf); + return new FragmentedTextDocument(document, fragment, fragment); } else { if (curOffInInput < mainBuf.length()) { // there's some text left between the last fragment and the end of the doc @@ -68,11 +74,15 @@ final class FragmentedDocBuilder { Chars remainder = mainBuf.slice(curOffInInput, remLen); lastFragment = new Fragment(lastFragment, remLen, remainder); } - return new FragmentedDocCursor(firstFragment); + return new FragmentedTextDocument(document, firstFragment, lastFragment); } } int inputOffsetAt(int outputOffset) { + return inputOffsetAt(outputOffset, firstFragment); + } + + static int inputOffsetAt(int outputOffset, Fragment firstFragment) { Fragment f = firstFragment; if (f == null) { return outputOffset; @@ -84,4 +94,138 @@ final class FragmentedDocBuilder { } return sum; } + + static class FragmentedTextDocument implements TextDocument { + + private final Fragment firstFragment; + private final Chars text; + private final TextDocument base; + + FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) { + this.firstFragment = firstFragment; + this.text = toChars(firstFragment, lastFragment); + this.base = base; + } + + private static Chars toChars(Fragment firstFragment, Fragment lastFragment) { + if (firstFragment == lastFragment) { + return firstFragment.getChars(); + } + StringBuilder sb = new StringBuilder(lastFragment.outEnd()); + Fragment f = firstFragment; + while (f.next != null) { + f.getChars().appendChars(sb); + f = f.next; + } + + return Chars.wrap(sb); + } + + @Override + public int translateOffset(int outputOffset) { + return base.translateOffset(inputOffsetAt(outputOffset, firstFragment)); + } + + @Override + public Chars getText() { + return text; + } + + @Override + public long getChecksum() { + return base.getChecksum(); + } + + @Override + public LanguageVersion getLanguageVersion() { + return base.getLanguageVersion(); + } + + @Override + public String getPathId() { + return base.getPathId(); + } + + @Override + public String getDisplayName() { + return base.getDisplayName(); + } + + @Override + public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { + throw new NotImplementedException("TODO"); + } + + @Override + public FileLocation toLocation(TextRegion region) { + return base.toLocation(TextRegion.fromBothOffsets(translateOffset(region.getStartOffset()), + translateOffset(region.getEndOffset()))); + } + + @Override + public void close() throws IOException { + base.close(); + } + } + + + static final class Fragment { + + private final Chars chars; + + final @Nullable Fragment prev; + @Nullable Fragment next; + + private final int inLength; + + Fragment(@Nullable Fragment prev, int inLength, Chars chars) { + this.chars = chars; + this.prev = prev; + this.inLength = inLength; + if (prev != null) { + prev.next = this; + } + } + + public Chars getChars() { + return chars; + } + + int outStart() { + return prev != null ? prev.outEnd() : 0; + } + + int outLen() { + return chars.length(); + } + + int outEnd() { + return outStart() + outLen(); + } + + int inStart() { + return prev != null ? prev.inEnd() : 0; + } + + int inLen() { + return inLength; + } + + int inEnd() { + return inStart() + inLen(); + } + + int outToIn(int outOffset) { + return inStart() + (outOffset - outStart()); + } + + boolean contains(int outOffset) { + return outStart() <= outOffset && outEnd() > outOffset; + } + + @Override + public String toString() { + return "Fragment[" + inStart() + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; + } + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java deleted file mode 100644 index 9f858b337a..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocCursor.java +++ /dev/null @@ -1,216 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc.io; - -import java.io.EOFException; - -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; - -import net.sourceforge.pmd.util.document.Chars; - -@SuppressWarnings("PMD.CompareObjectsWithEquals") -final class FragmentedDocCursor { - - private static final EOFException EOF = new EOFException(); - - private Fragment cur; - private int curOutPos; - - private Fragment mark; - private int markOutPos; - - // We can reuse a stringbuilder - // This means, the byte[] buffer is shared, and doesn't need to be - // resized so often. StringBuilder::toString always does an array - // copy anyway. - - // This also means that if we encounter some non-latin1 characters, - // all the following tokens will be coded as utf16. The string - // builder is only used for tokens that span several file fragments - // though, so that is a rare situation - private final StringBuilder strBuilder = new StringBuilder(); - - FragmentedDocCursor(Fragment firstFragment) { - cur = firstFragment; - mark = firstFragment; - checkAssertions(); - } - - private StringBuilder freshStrBuilder(int minCap) { - strBuilder.setLength(0); - strBuilder.ensureCapacity(minCap); - return strBuilder; - } - - char next() throws EOFException { - Fragment f = cur; - - if (curOutPos >= f.outEnd()) { - // slow path, fragment transition - do { - f = f.next; - } while (f != null && curOutPos >= f.outEnd()); - - if (f == null) { - throw EOF; - } - cur = f; - } - - return f.charAt(curOutPos++); - } - - void backup(final int amount) { - if (amount == 1 && cur.outStart() < curOutPos) { - // fast path - curOutPos--; - return; - } - - final int posToRetreatTo = curOutPos - amount; - this.curOutPos = posToRetreatTo; - this.cur = findBackwards(posToRetreatTo); - checkAssertions(); - } - - - void appendMarkSuffix(StringBuilder sb, final int suffixLen) { - assert suffixLen <= markLength() : "Suffix is greater than the mark length? " + suffixLen + " > " + markLength(); - - if (cur == mark || cur.outStart() <= curOutPos - suffixLen) { - // entire suffix is in the last fragment, fast path - cur.chars.appendChars(sb, curOutPos - cur.outStart() - suffixLen, suffixLen); - } else { - int suffixStart = curOutPos - suffixLen; - Fragment f = findBackwards(suffixStart); - sb.ensureCapacity(sb.length() + suffixLen); - appendUntilCurPos(f, sb, suffixStart); - } - } - - Chars getMarkImage() { - Fragment f = mark; - if (f == cur) { // same fragment, this is the fast path - return f.chars.slice(markOutPos - f.outStart(), markLength()); - } - - StringBuilder sb = freshStrBuilder(markLength()); - appendUntilCurPos(f, sb, markOutPos); - assert sb.length() == markLength() : sb + " should have length " + markLength(); - return Chars.wrap(sb); - } - - private void appendUntilCurPos(Fragment f, StringBuilder sb, int startOutPos) { - assert f != null && f != cur; // this won't work otherwise - assert f.outStart() <= startOutPos && startOutPos < f.outEnd(); - - // append the suffix of the first fragment after the start pos - f.appendAbs(sb, startOutPos, f.outEnd()); - f = f.next; - while (f != cur) { - // append whole fragments - f.appendAbs(sb, f.outStart(), f.outEnd()); - f = f.next; - } - // append the prefix of the last fragment until the current pos - f.appendAbs(sb, f.outStart(), curOutPos); - } - - - private void checkAssertions() { - assert mark != null && cur != null : "Null mark or current fragment"; - assert cur.outStart() >= mark.outStart() : "Mark is after the current fragment"; - } - - // find the fragment that contains the given out offset - private @NonNull Fragment findBackwards(int posToRetreatTo) { - Fragment f = cur; - while (f != null && f.outStart() > posToRetreatTo) { - f = f.prev; - } - if (f == null) { - throw new IllegalArgumentException("Cannot retreat to " + posToRetreatTo); - } - return f; - } - - void mark() { - mark = cur; - markOutPos = curOutPos; - } - - int markInOffset() { - return mark.outToIn(markOutPos); - } - - int curInOffset() { - return cur.outToIn(curOutPos); - } - - int markLength() { - return curOutPos - markOutPos; - } - - static final class Fragment { - - private final Chars chars; - - final @Nullable Fragment prev; - @Nullable Fragment next; - - private final int inLength; - - Fragment(@Nullable Fragment prev, int inLength, Chars chars) { - this.chars = chars; - this.prev = prev; - this.inLength = inLength; - if (prev != null) { - prev.next = this; - } - } - - void appendAbs(StringBuilder sb, int absOffset, int absEndOffset) { - chars.appendChars(sb, absOffset - outStart(), absEndOffset - absOffset); - } - - char charAt(int absPos) { - return chars.charAt(absPos - outStart()); - } - - int outStart() { - return prev != null ? prev.outEnd() : 0; - } - - int outLen() { - return chars.length(); - } - - int outEnd() { - return outStart() + outLen(); - } - - int inStart() { - return prev != null ? prev.inEnd() : 0; - } - - int inLen() { - return inLength; - } - - int inEnd() { - return inStart() + inLen(); - } - - int outToIn(int outOffset) { - return inStart() + (outOffset - outStart()); - } - - @Override - public String toString() { - return "Fragment[" + inStart() + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; - } - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index a2db0ea2dd..151ed21e5a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -102,8 +102,13 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { } @Override - public TextFileContent getContent() { - return content; + public long getChecksum() { + return content.getCheckSum(); + } + + @Override + public int translateOffset(int outOffset) { + return outOffset; } private static final String NOT_IN_RANGE = "Region [start=%d, end=%d[ is not in range of this document (length %d)"; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index ef5cf324a7..4c8bf577fb 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -54,9 +54,7 @@ public interface TextDocument extends Closeable { * * @see TextFileContent#getNormalizedText() */ - default Chars getText() { - return getContent().getNormalizedText(); - } + Chars getText(); /** * Returns a region of the {@linkplain #getText() text} as a character sequence. @@ -67,9 +65,13 @@ public interface TextDocument extends Closeable { /** - * Returns the current contents of the text file. See also {@link #getText()}. + * Returns a checksum for the contents of the file. + * + * @see TextFileContent#getCheckSum() */ - TextFileContent getContent(); + long getChecksum(); + + int translateOffset(int outOffset); /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextRegion.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextRegion.java index 3570e47ca9..8e6a8968a9 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextRegion.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextRegion.java @@ -183,6 +183,15 @@ public final class TextRegion implements Comparable { return new TextRegion(startOffset, length); } + /** + * Builds a new region from offset and length. + * + * @throws AssertionError If either parameter is negative + */ + public static TextRegion caretAt(int offset) { + return fromOffsetLength(offset, 0); + } + /** * Builds a new region from start and end offset. * From 155a895a937950fe28b39fc0766d76475e252b9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 05:16:32 +0200 Subject: [PATCH 042/136] Move charstream --- .../pmd/cpd/internal/JavaCCTokenizer.java | 2 +- .../ast/impl/javacc/{io => }/CharStream.java | 13 +++++------ .../pmd/lang/ast/impl/javacc/JavaccToken.java | 1 - .../ast/impl/javacc/JavaccTokenDocument.java | 10 +++++++- .../ast/impl/javacc/JjtreeParserAdapter.java | 1 - .../ast/impl/javacc/io/EscapeAwareReader.java | 3 ++- .../impl/javacc/io/FragmentedDocBuilder.java | 23 ++++++++++++------- .../pmd/util/document/RootTextDocument.java | 5 ++++ .../impl/javacc/io/CharStreamImplTest.java | 1 + .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 2 +- .../pmd/cpd/CppCharStreamTest.java | 2 +- .../sourceforge/pmd/cpd/JavaTokenizer.java | 2 +- .../pmd/lang/java/ast/JavaParser.java | 2 +- .../pmd/lang/java/ast/JavaTokenDocument.java | 2 +- .../pmd/cpd/EcmascriptTokenizer.java | 2 +- .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 2 +- .../pmd/lang/jsp/ast/JspParser.java | 2 +- .../sourceforge/pmd/cpd/MatlabTokenizer.java | 2 +- .../pmd/cpd/ModelicaTokenizer.java | 2 +- .../pmd/lang/modelica/ast/ModelicaParser.java | 2 +- .../pmd/cpd/ObjectiveCTokenizer.java | 2 +- .../sourceforge/pmd/cpd/PLSQLTokenizer.java | 2 +- .../pmd/lang/plsql/ast/PLSQLParser.java | 2 +- .../sourceforge/pmd/cpd/PythonTokenizer.java | 2 +- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 2 +- .../sourceforge/pmd/lang/vf/ast/VfParser.java | 2 +- .../sourceforge/pmd/lang/vm/ast/VmParser.java | 2 +- 27 files changed, 57 insertions(+), 38 deletions(-) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io => }/CharStream.java (92%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 9e54955e00..b88f267dfa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -16,7 +16,7 @@ import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.FileAnalysisException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java similarity index 92% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index 68e3264f6b..084a82ac31 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -2,13 +2,13 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.EOFException; import java.io.IOException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextDocument; @@ -28,9 +28,9 @@ public final class CharStream { private int curOffset; private int markOffset = -1; - private CharStream(JavaccTokenDocument tokenDoc, TextDocument textDoc) { + private CharStream(JavaccTokenDocument tokenDoc) { this.tokenDoc = tokenDoc; - this.textDoc = textDoc; + this.textDoc = tokenDoc.getTextDocument(); this.chars = textDoc.getText(); this.useMarkSuffix = tokenDoc.useMarkSuffix(); } @@ -39,9 +39,8 @@ public final class CharStream { * Create a new char stream for the given document. */ public static CharStream create(JavaccTokenDocument doc) throws IOException, MalformedSourceException { - try (EscapeAwareReader reader = doc.newReader(doc.getTextDocument().getText())) { - return new CharStream(doc, reader.translate(doc.getTextDocument())); - } + doc.translate(); + return new CharStream(doc); } /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index ab9855d46f..d4f433188b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -5,7 +5,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import net.sourceforge.pmd.lang.ast.GenericToken; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextRegion; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 21b023a72e..6e72604602 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -4,12 +4,14 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; +import java.io.IOException; + import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -22,6 +24,7 @@ public class JavaccTokenDocument extends TokenDocument { final StringPool stringPool = new StringPool(); private JavaccToken first; + private TextDocument translatedDocument; public JavaccTokenDocument(TextDocument textDocument) { super(textDocument); @@ -47,6 +50,11 @@ public class JavaccTokenDocument extends TokenDocument { return new EscapeAwareReader(text); } + final void translate() throws IOException, MalformedSourceException { + try (EscapeAwareReader reader = newReader(getTextDocument().getText())) { + translatedDocument = reader.translate(getTextDocument()); + } + } /** * Open the document. This is only meant to be used by a Javacc-generated diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index f2a0d0b90c..c37da63d78 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -10,7 +10,6 @@ import net.sourceforge.pmd.lang.Parser; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.RootNode; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; import net.sourceforge.pmd.util.document.TextDocument; /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java index 4a12aa2cae..a705e83482 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java @@ -13,6 +13,7 @@ import java.io.Reader; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.internal.util.AssertionUtil; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -60,7 +61,7 @@ public class EscapeAwareReader extends Reader { /** * Translate all the input in the buffer. This is fed to a cursor initialized to zero. */ - TextDocument translate(TextDocument source) throws IOException, MalformedSourceException { + public TextDocument translate(TextDocument source) throws IOException, MalformedSourceException { readUnchecked(null, 0, Integer.MAX_VALUE); return escapes.build(source); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 006767e3f1..91e7254672 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -7,7 +7,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.IOException; -import org.apache.commons.lang3.NotImplementedException; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.LanguageVersion; @@ -61,12 +60,11 @@ final class FragmentedDocBuilder { this.curOffInInput = endInInput; } - TextDocument build(TextDocument document) { + TextDocument build(TextDocument original) { if (firstFragment == null) { // No deltas in whole document, there's a single fragment // This is the case for > 97% of Java files (source: OpenJDK) - Fragment fragment = new Fragment(null, mainBuf.length(), mainBuf); - return new FragmentedTextDocument(document, fragment, fragment); + return original; } else { if (curOffInInput < mainBuf.length()) { // there's some text left between the last fragment and the end of the doc @@ -74,7 +72,7 @@ final class FragmentedDocBuilder { Chars remainder = mainBuf.slice(curOffInInput, remLen); lastFragment = new Fragment(lastFragment, remLen, remainder); } - return new FragmentedTextDocument(document, firstFragment, lastFragment); + return new FragmentedTextDocument(original, firstFragment, lastFragment); } } @@ -123,6 +121,8 @@ final class FragmentedDocBuilder { @Override public int translateOffset(int outputOffset) { + // todo this would be pretty slow when there are many escapes + // we could check save the fragment last accessed and return base.translateOffset(inputOffsetAt(outputOffset, firstFragment)); } @@ -153,7 +153,7 @@ final class FragmentedDocBuilder { @Override public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { - throw new NotImplementedException("TODO"); + return base.createLineRange(startLineInclusive, endLineInclusive); } @Override @@ -177,6 +177,8 @@ final class FragmentedDocBuilder { @Nullable Fragment next; private final int inLength; + private final int outStart; + private final int inStart; Fragment(@Nullable Fragment prev, int inLength, Chars chars) { this.chars = chars; @@ -184,6 +186,11 @@ final class FragmentedDocBuilder { this.inLength = inLength; if (prev != null) { prev.next = this; + this.outStart = prev.outEnd(); + this.inStart = prev.inEnd(); + } else { + this.outStart = 0; + this.inStart = 0; } } @@ -192,7 +199,7 @@ final class FragmentedDocBuilder { } int outStart() { - return prev != null ? prev.outEnd() : 0; + return outStart; } int outLen() { @@ -204,7 +211,7 @@ final class FragmentedDocBuilder { } int inStart() { - return prev != null ? prev.inEnd() : 0; + return inStart; } int inLen() { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index 151ed21e5a..da3a3f9f92 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -62,6 +62,11 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { backend.close(); } + @Override + public Chars getText() { + return content.getNormalizedText(); + } + @Override public FileLocation toLocation(TextRegion region) { checkInRange(region); diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index 1803f3e419..ab69c2320d 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -16,6 +16,7 @@ import org.junit.rules.ExpectedException; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersion; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index dc764c9939..f824c5f5cd 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -13,7 +13,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 42eb5c1dd9..50d3c3c409 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -11,7 +11,7 @@ import java.io.IOException; import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; import net.sourceforge.pmd.util.document.TextFile; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index c58e86b136..a9daf1c1e1 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -15,7 +15,7 @@ import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index 2f77e0bfb0..716e625b2f 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.lang.java.ast; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 56f8690c06..159a5ee243 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -16,7 +16,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.util.document.Chars; diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index 959d446fe6..d66d74949f 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ecmascript5.ast.Ecmascript5TokenKinds; diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index c5ac10410e..e160b67b19 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -8,7 +8,7 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java index fe8a329dcc..4252bf16d4 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java @@ -9,7 +9,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.document.TextDocument; /** diff --git a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java index 17b0ba9003..9459c44696 100644 --- a/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java +++ b/pmd-matlab/src/main/java/net/sourceforge/pmd/cpd/MatlabTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.matlab.ast.MatlabTokenKinds; diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java index eabfc401d8..3258a3cda7 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/cpd/ModelicaTokenizer.java @@ -7,7 +7,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.modelica.ast.ModelicaTokenKinds; diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java index 3cf646d962..058941d176 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java @@ -4,7 +4,7 @@ package net.sourceforge.pmd.lang.modelica.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java index 1e4bfb3e5a..acccfcd24a 100644 --- a/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java +++ b/pmd-objectivec/src/main/java/net/sourceforge/pmd/cpd/ObjectiveCTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.objectivec.ast.ObjectiveCTokenKinds; diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index 72a90a80b2..32dbbf94c5 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -9,7 +9,7 @@ import java.util.Properties; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.plsql.ast.PLSQLTokenKinds; public class PLSQLTokenizer extends JavaCCTokenizer { diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java index ab7e455512..e2beed5425 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java @@ -9,7 +9,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.document.TextDocument; public class PLSQLParser extends JjtreeParserAdapter { diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index de6d1b2b01..70755980ea 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -10,7 +10,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds; diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 80d768ff7a..56a6772478 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java index f27ad84dd0..b740cc8c63 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java @@ -6,7 +6,7 @@ package net.sourceforge.pmd.lang.vf.ast; import org.checkerframework.checker.nullness.qual.Nullable; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java index e555891db1..9826dc367c 100644 --- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java +++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java @@ -10,7 +10,7 @@ import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.document.TextDocument; /** From a11f45e511e6c36b98254d4af017d6fc910b2598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 06:07:39 +0200 Subject: [PATCH 043/136] Cleanup spec --- javacc-wrapper.xml | 2 +- .../pmd/cpd/internal/AntlrTokenizer.java | 2 +- .../pmd/cpd/internal/JavaCCTokenizer.java | 2 +- .../ast/impl/javacc/AbstractJjtreeNode.java | 2 +- .../pmd/lang/ast/impl/javacc/CharStream.java | 15 ++++--- .../ast/impl/javacc/JavaccTokenDocument.java | 6 +++ .../impl/javacc/io/FragmentedDocBuilder.java | 41 +++++++++++-------- .../pmd/util/document/RootTextDocument.java | 10 +++++ .../pmd/util/document/TextDocument.java | 38 +++++++++++++++-- .../pmd/util/document/TextFileBuilder.java | 1 + .../impl/javacc/io/CharStreamImplTest.java | 7 ++-- .../pmd/util/document/TextDocumentTest.java | 6 +-- pmd-cpp/etc/grammar/Cpp.jj | 2 +- .../pmd/lang/java/ast/ASTCompilationUnit.java | 7 ++-- .../pmd/lang/java/ast/JavaParser.java | 2 +- pmd-javascript/etc/grammar/Ecmascript5.jj | 2 +- pmd-jsp/etc/grammar/Jsp.jjt | 2 +- pmd-matlab/etc/grammar/Matlab.jj | 2 +- pmd-modelica/etc/grammar/Modelica.jjt | 2 +- pmd-objectivec/etc/grammar/ObjectiveC.jj | 2 +- pmd-plsql/etc/grammar/PLSQL.jjt | 2 +- pmd-python/etc/grammar/Python.jj | 2 +- pmd-visualforce/etc/grammar/Vf.jjt | 2 +- pmd-vm/etc/grammar/Vm.jjt | 2 +- 24 files changed, 110 insertions(+), 51 deletions(-) diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index b303e31e6d..60d4d70158 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -407,7 +407,7 @@ public final class ${token-constants-name} \{${line.separator} * be used as a basis for a CPD Tokenizer. */ @net.sourceforge.pmd.annotation.InternalApi - public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.io.CharStream cs) { + public static net.sourceforge.pmd.lang.TokenManager<%%%API_PACK%%%.impl.javacc.JavaccToken> newTokenManager(%%%API_PACK%%%.impl.javacc.CharStream cs) { return new %%%TOKEN_MGR_NAME%%%(cs); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java index f0659b952b..427cb26325 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/AntlrTokenizer.java @@ -18,8 +18,8 @@ import net.sourceforge.pmd.cpd.Tokens; import net.sourceforge.pmd.cpd.token.AntlrTokenFilter; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrToken; import net.sourceforge.pmd.lang.ast.impl.antlr4.AntlrTokenManager; -import net.sourceforge.pmd.util.document.TextDocument; import net.sourceforge.pmd.util.document.CpdCompat; +import net.sourceforge.pmd.util.document.TextDocument; /** * Generic implementation of a {@link Tokenizer} useful to any Antlr grammar. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index b88f267dfa..f4711d20a6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -14,9 +14,9 @@ import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.FileAnalysisException; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java index 6b3dddbe9e..a2a5035948 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java @@ -51,7 +51,7 @@ public abstract class AbstractJjtreeNode, N e @Override public Chars getText() { - return getTextDocument().sliceText(getTextRegion()); + return getTextDocument().sliceOriginalText(getTextRegion()); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index 084a82ac31..040f3887f1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -26,7 +26,7 @@ public final class CharStream { private final Chars chars; private final boolean useMarkSuffix; private int curOffset; - private int markOffset = -1; + private int markOffset; private CharStream(JavaccTokenDocument tokenDoc) { this.tokenDoc = tokenDoc; @@ -120,9 +120,12 @@ public final class CharStream { * be used again as the prefix of the next token. * * @throws AssertionError If the requested amount is greater than the - * number of read chars + * length of the mark */ public void backup(int amount) { + if (amount > markLen()) { + throw new IllegalArgumentException(); + } curOffset -= amount; } @@ -149,15 +152,15 @@ public final class CharStream { } - /** Returns the start offset of the current token (in the original source), inclusive. */ + /** Returns the start offset of the current token (in the translated source), inclusive. */ public int getStartOffset() { - return textDoc.translateOffset(markOffset); + return markOffset; } - /** Returns the end offset of the current token (in the original source), exclusive. */ + /** Returns the end offset of the current token (in the translated source), exclusive. */ public int getEndOffset() { - return textDoc.translateOffset(curOffset); + return curOffset; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 6e72604602..8d482f3340 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -50,6 +50,12 @@ public class JavaccTokenDocument extends TokenDocument { return new EscapeAwareReader(text); } + @Override + public TextDocument getTextDocument() { + return translatedDocument == null ? super.getTextDocument() + : translatedDocument; + } + final void translate() throws IOException, MalformedSourceException { try (EscapeAwareReader reader = newReader(getTextDocument().getText())) { translatedDocument = reader.translate(getTextDocument()); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java index 91e7254672..68516ad79c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.IOException; +import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.LanguageVersion; @@ -80,49 +81,44 @@ final class FragmentedDocBuilder { return inputOffsetAt(outputOffset, firstFragment); } - static int inputOffsetAt(int outputOffset, Fragment firstFragment) { + static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment) { Fragment f = firstFragment; if (f == null) { return outputOffset; } - int sum = outputOffset; - while (f != null && f.inStart() < sum) { - sum += f.inLen(); + while (f.next != null && f.inEnd() < outputOffset) { f = f.next; } - return sum; + return f.outToIn(outputOffset); } - static class FragmentedTextDocument implements TextDocument { + static final class FragmentedTextDocument implements TextDocument { private final Fragment firstFragment; private final Chars text; private final TextDocument base; FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) { + assert firstFragment != lastFragment; // NOPMD this.firstFragment = firstFragment; this.text = toChars(firstFragment, lastFragment); this.base = base; } private static Chars toChars(Fragment firstFragment, Fragment lastFragment) { - if (firstFragment == lastFragment) { - return firstFragment.getChars(); - } StringBuilder sb = new StringBuilder(lastFragment.outEnd()); Fragment f = firstFragment; - while (f.next != null) { + while (f != null) { f.getChars().appendChars(sb); f = f.next; } - return Chars.wrap(sb); } @Override public int translateOffset(int outputOffset) { - // todo this would be pretty slow when there are many escapes - // we could check save the fragment last accessed and + // todo this would be pretty slow when we're in the middle of some escapes + // we could check save the fragment last accessed to speed it up, and look forwards & backwards return base.translateOffset(inputOffsetAt(outputOffset, firstFragment)); } @@ -156,10 +152,20 @@ final class FragmentedDocBuilder { return base.createLineRange(startLineInclusive, endLineInclusive); } + @Override + public Chars sliceOriginalText(TextRegion region) { + return base.sliceOriginalText(translateRegion(region)); + } + @Override public FileLocation toLocation(TextRegion region) { - return base.toLocation(TextRegion.fromBothOffsets(translateOffset(region.getStartOffset()), - translateOffset(region.getEndOffset()))); + return base.toLocation(translateRegion(region)); + } + + @Override + public @NonNull TextRegion translateRegion(TextRegion region) { + return TextRegion.fromBothOffsets(translateOffset(region.getStartOffset()), + translateOffset(region.getEndOffset())); } @Override @@ -168,7 +174,10 @@ final class FragmentedDocBuilder { } } - + /** + * A delta from the original text to the translated text. This maps + * a region of the original document to some new characters. + */ static final class Fragment { private final Chars chars; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index da3a3f9f92..a48a62ad47 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -116,6 +116,16 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { return outOffset; } + @Override + public TextRegion translateRegion(TextRegion region) { + return region; + } + + @Override + public Chars sliceOriginalText(TextRegion region) { + return getText().subSequence(region.getStartOffset(), region.getEndOffset()); + } + private static final String NOT_IN_RANGE = "Region [start=%d, end=%d[ is not in range of this document (length %d)"; private static final String INVALID_LINE_RANGE = "Line range %d..%d is not in range of this document (%d lines) (line numbers are 1-based)"; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 4c8bf577fb..b0ecdf7344 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -57,11 +57,17 @@ public interface TextDocument extends Closeable { Chars getText(); /** - * Returns a region of the {@linkplain #getText() text} as a character sequence. + * Returns a slice of the original text. Note that this is not the + * same as {@code getText().subsequence}, as if this document has + * translated escapes, the returned char slice will contain the + * untranslated escapes, whereas {@link #getText()} would return + * the translated characters. + * + * @param region A region, in the coordinate system of this document + * + * @return The slice of the original text that corresponds to the region */ - default Chars sliceText(TextRegion region) { - return getText().subSequence(region.getStartOffset(), region.getEndOffset()); - } + Chars sliceOriginalText(TextRegion region); /** @@ -71,8 +77,31 @@ public interface TextDocument extends Closeable { */ long getChecksum(); + + /** + * Returns the input offset for the given output offset. This maps + * back an offset in the coordinate system of this document, to the + * coordinate system of the original document. + * + * @param outOffset Output offset + * + * @return Input offset + */ int translateOffset(int outOffset); + /** + * Translate a region given in the the coordinate system of this + * document, to the coordinate system of the original document. + * This works as if creating a new region with both start and end + * offsets translated through {@link #translateOffset(int)}. The + * returned region may have a different length. + * + * @param region Output region + * + * @return Input region + */ + TextRegion translateRegion(TextRegion region); + /** * Returns the length in characters of the {@linkplain #getText() text}. @@ -152,6 +181,7 @@ public interface TextDocument extends Closeable { * * @see TextFile#forCharSeq(CharSequence, String, LanguageVersion) */ + @SuppressWarnings("PMD.CloseResource") static TextDocument readOnlyString(@NonNull CharSequence source, @NonNull String filename, @NonNull LanguageVersion lv) { TextFile textFile = TextFile.forCharSeq(source, filename, lv); try { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextFileBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextFileBuilder.java index ea887e6ff4..a30c1dce5f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextFileBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextFileBuilder.java @@ -16,6 +16,7 @@ import net.sourceforge.pmd.lang.LanguageVersion; /** * A builder for a new text file. */ +@SuppressWarnings("PMD.MissingStaticMethodInNonInstantiatableClass") public abstract class TextFileBuilder { protected final LanguageVersion languageVersion; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index ab69c2320d..f26f6fd7d1 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -53,8 +53,8 @@ public class CharStreamImplTest { try { stream.readChar(); } catch (Exception e) { - assertEquals(stream.getStartOffset(), 0); - assertEquals(stream.getEndOffset(), 0); + assertEquals(0, stream.getStartOffset()); + assertEquals(0, stream.getEndOffset()); throw e; } } @@ -162,8 +162,9 @@ public class CharStreamImplTest { assertEquals('c', stream.markTokenStart()); assertEquals('d', stream.readChar()); + stream.backup(2); // ok expect.expect(IllegalArgumentException.class); - stream.backup(10); + stream.backup(1); } @Test diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/util/document/TextDocumentTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/util/document/TextDocumentTest.java index fab67f41e4..242687bc14 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/util/document/TextDocumentTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/util/document/TextDocumentTest.java @@ -43,7 +43,7 @@ public class TextDocumentTest { TextDocument doc = TextDocument.readOnlyString("bonjour\ntristesse", dummyVersion); TextRegion region = TextRegion.fromOffsetLength(0, "bonjour\n".length()); - assertEquals("bonjour\n", doc.sliceText(region).toString()); + assertEquals("bonjour\n", doc.sliceOriginalText(region).toString()); FileLocation withLines = doc.toLocation(region); assertEquals(1, withLines.getBeginLine()); @@ -60,7 +60,7 @@ public class TextDocumentTest { // We consider it's part of the next line TextRegion region = TextRegion.fromOffsetLength("bonjour\n".length(), 0); - assertEquals("", doc.sliceText(region).toString()); + assertEquals("", doc.sliceOriginalText(region).toString()); FileLocation withLines = doc.toLocation(region); @@ -78,7 +78,7 @@ public class TextDocumentTest { TextRegion region = TextRegion.fromOffsetLength("bonjour".length(), 1); - assertEquals("\n", doc.sliceText(region).toString()); + assertEquals("\n", doc.sliceOriginalText(region).toString()); FileLocation withLines = doc.toLocation(region); diff --git a/pmd-cpp/etc/grammar/Cpp.jj b/pmd-cpp/etc/grammar/Cpp.jj index f292dd019c..eed7cc58cf 100644 --- a/pmd-cpp/etc/grammar/Cpp.jj +++ b/pmd-cpp/etc/grammar/Cpp.jj @@ -32,7 +32,7 @@ options { PARSER_BEGIN(CppParserImpl) package net.sourceforge.pmd.lang.cpp.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public final class CppParserImpl { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java index 9caac9c173..2f049916cf 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java @@ -12,7 +12,6 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.annotation.InternalApi; -import net.sourceforge.pmd.lang.Parser.ParserTask; import net.sourceforge.pmd.lang.ast.NodeStream; import net.sourceforge.pmd.lang.ast.RootNode; import net.sourceforge.pmd.lang.ast.impl.GenericNode; @@ -42,11 +41,11 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ja return doc; } - void addTaskInfo(ParserTask task) { - this.doc = task.getTextDocument(); + void addTaskInfo(TextDocument translatedDoc) { + this.doc = translatedDoc; } - void setComments(List comments) { + void setComments(List comments) { this.comments = comments; } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index 716e625b2f..fc3e890532 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -40,7 +40,7 @@ public class JavaParser extends JjtreeParserAdapter { ASTCompilationUnit acu = parser.CompilationUnit(); acu.setNoPmdComments(parser.getSuppressMap()); - acu.addTaskInfo(task); + acu.addTaskInfo(cs.getTokenDocument().getTextDocument()); // this is the translated document != task.getTextDocument() checker.check(acu); return acu; } diff --git a/pmd-javascript/etc/grammar/Ecmascript5.jj b/pmd-javascript/etc/grammar/Ecmascript5.jj index a1b77891ce..a9cf14bcbb 100644 --- a/pmd-javascript/etc/grammar/Ecmascript5.jj +++ b/pmd-javascript/etc/grammar/Ecmascript5.jj @@ -15,7 +15,7 @@ options { PARSER_BEGIN(Ecmascript5ParserImpl) package net.sourceforge.pmd.lang.ecmascript5.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class Ecmascript5ParserImpl { diff --git a/pmd-jsp/etc/grammar/Jsp.jjt b/pmd-jsp/etc/grammar/Jsp.jjt index ca66ff9571..9e4483a441 100644 --- a/pmd-jsp/etc/grammar/Jsp.jjt +++ b/pmd-jsp/etc/grammar/Jsp.jjt @@ -30,7 +30,7 @@ options { PARSER_BEGIN(JspParserImpl) package net.sourceforge.pmd.lang.jsp.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; diff --git a/pmd-matlab/etc/grammar/Matlab.jj b/pmd-matlab/etc/grammar/Matlab.jj index 6220de48c8..3ff1c8b27c 100644 --- a/pmd-matlab/etc/grammar/Matlab.jj +++ b/pmd-matlab/etc/grammar/Matlab.jj @@ -21,7 +21,7 @@ options { PARSER_BEGIN(MatlabParserImpl) package net.sourceforge.pmd.lang.matlab.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class MatlabParserImpl { diff --git a/pmd-modelica/etc/grammar/Modelica.jjt b/pmd-modelica/etc/grammar/Modelica.jjt index 9d2cdfa526..e2e7d63121 100644 --- a/pmd-modelica/etc/grammar/Modelica.jjt +++ b/pmd-modelica/etc/grammar/Modelica.jjt @@ -49,7 +49,7 @@ options { PARSER_BEGIN(ModelicaParserImpl) package net.sourceforge.pmd.lang.modelica.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; class ModelicaParserImpl { diff --git a/pmd-objectivec/etc/grammar/ObjectiveC.jj b/pmd-objectivec/etc/grammar/ObjectiveC.jj index c4679634a1..171ca581fa 100644 --- a/pmd-objectivec/etc/grammar/ObjectiveC.jj +++ b/pmd-objectivec/etc/grammar/ObjectiveC.jj @@ -21,7 +21,7 @@ package net.sourceforge.pmd.lang.objectivec.ast; import java.io.*; import java.util.*; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; /** diff --git a/pmd-plsql/etc/grammar/PLSQL.jjt b/pmd-plsql/etc/grammar/PLSQL.jjt index a44efb486a..090f90e235 100644 --- a/pmd-plsql/etc/grammar/PLSQL.jjt +++ b/pmd-plsql/etc/grammar/PLSQL.jjt @@ -154,7 +154,7 @@ package net.sourceforge.pmd.lang.plsql.ast; import java.io.*; import net.sourceforge.pmd.lang.ast.Node; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class PLSQLParserImpl { diff --git a/pmd-python/etc/grammar/Python.jj b/pmd-python/etc/grammar/Python.jj index cba9a4fce6..8087042e7c 100644 --- a/pmd-python/etc/grammar/Python.jj +++ b/pmd-python/etc/grammar/Python.jj @@ -17,7 +17,7 @@ PARSER_BEGIN(PythonParserImpl) package net.sourceforge.pmd.lang.python.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class PythonParserImpl { diff --git a/pmd-visualforce/etc/grammar/Vf.jjt b/pmd-visualforce/etc/grammar/Vf.jjt index b123075211..2af403e0cb 100644 --- a/pmd-visualforce/etc/grammar/Vf.jjt +++ b/pmd-visualforce/etc/grammar/Vf.jjt @@ -13,7 +13,7 @@ options { PARSER_BEGIN(VfParserImpl) package net.sourceforge.pmd.lang.vf.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; public class VfParserImpl { diff --git a/pmd-vm/etc/grammar/Vm.jjt b/pmd-vm/etc/grammar/Vm.jjt index 5dc623deb7..678ed8de31 100644 --- a/pmd-vm/etc/grammar/Vm.jjt +++ b/pmd-vm/etc/grammar/Vm.jjt @@ -46,7 +46,7 @@ import java.util.Map; import org.apache.commons.lang3.StringUtils; import net.sourceforge.pmd.lang.ast.TokenMgrError; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; From a01f1a35742578aca09914329bde24eb6a5a186a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 07:10:25 +0200 Subject: [PATCH 044/136] Stop extending Reader --- .../pmd/lang/ast/impl/javacc/CharStream.java | 5 +- .../ast/impl/javacc/JavaccTokenDocument.java | 14 +- .../ast/impl/javacc/JjtreeParserAdapter.java | 8 +- ...er.java => BackslashEscapeTranslator.java} | 7 +- ...AwareReader.java => EscapeTranslator.java} | 94 ++----- .../impl/javacc/io/FragmentedDocBuilder.java | 247 ------------------ ...eReader.java => JavaEscapeTranslator.java} | 7 +- .../sourceforge/pmd/util/document/Chars.java | 8 + .../util/document/FragmentedDocBuilder.java | 81 ++++++ .../util/document/FragmentedTextDocument.java | 180 +++++++++++++ .../pmd/util/document/RootTextDocument.java | 6 +- .../pmd/util/document/TextDocument.java | 20 +- .../impl/javacc/io/CharStreamImplTest.java | 12 +- .../impl/javacc/io/JavaEscapeReaderTest.java | 18 +- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 13 +- ...peReader.java => CppEscapeTranslator.java} | 7 +- .../pmd/lang/java/ast/JavaTokenDocument.java | 13 +- .../pmd/lang/java/ast/ParserCornersTest.java | 1 + .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 11 +- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 12 +- 20 files changed, 381 insertions(+), 383 deletions(-) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/{BackslashEscapeReader.java => BackslashEscapeTranslator.java} (90%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/{EscapeAwareReader.java => EscapeTranslator.java} (57%) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/{JavaEscapeReader.java => JavaEscapeTranslator.java} (91%) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java rename pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/{CppEscapeReader.java => CppEscapeTranslator.java} (81%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index 040f3887f1..da881ef53b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.EOFException; -import java.io.IOException; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.Chars; @@ -38,8 +37,8 @@ public final class CharStream { /** * Create a new char stream for the given document. */ - public static CharStream create(JavaccTokenDocument doc) throws IOException, MalformedSourceException { - doc.translate(); + public static CharStream create(JavaccTokenDocument doc) throws MalformedSourceException { + doc.doTranslate(); return new CharStream(doc); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 8d482f3340..67ccbf63d9 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -4,15 +4,11 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import java.io.IOException; - import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; -import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -46,8 +42,8 @@ public class JavaccTokenDocument extends TokenDocument { * * @param text Source doc */ - public EscapeAwareReader newReader(Chars text) { - return new EscapeAwareReader(text); + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + return text; } @Override @@ -56,9 +52,9 @@ public class JavaccTokenDocument extends TokenDocument { : translatedDocument; } - final void translate() throws IOException, MalformedSourceException { - try (EscapeAwareReader reader = newReader(getTextDocument().getText())) { - translatedDocument = reader.translate(getTextDocument()); + final void doTranslate() throws MalformedSourceException { + if (translatedDocument == null) { + translatedDocument = translate(super.getTextDocument()); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index c37da63d78..71a9da0a25 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -4,12 +4,10 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; -import java.io.IOException; - import net.sourceforge.pmd.lang.Parser; +import net.sourceforge.pmd.lang.ast.FileAnalysisException; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.RootNode; -import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -34,9 +32,7 @@ public abstract class JjtreeParserAdapter implements Parser try { CharStream charStream = CharStream.create(doc); return parseImpl(charStream, task); - } catch (IOException e) { - throw new TokenMgrError(-1, -1, task.getFileDisplayName(), "IO error", e); - } catch (TokenMgrError tme) { + } catch (FileAnalysisException tme) { throw tme.setFileName(task.getFileDisplayName()); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeTranslator.java similarity index 90% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeTranslator.java index 9bed9435d9..a5a9a68064 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeTranslator.java @@ -7,11 +7,12 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; /** * A base class for readers that handle escapes starting with a backslash. */ -public abstract class BackslashEscapeReader extends EscapeAwareReader { +public abstract class BackslashEscapeTranslator extends EscapeTranslator { private static final char BACKSLASH = '\\'; @@ -24,8 +25,8 @@ public abstract class BackslashEscapeReader extends EscapeAwareReader { private int savedNotEscapeSpecialEnd = Integer.MAX_VALUE; - public BackslashEscapeReader(Chars input) { - super(input); + public BackslashEscapeTranslator(TextDocument builder) { + super(builder); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java similarity index 57% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java index a705e83482..ee9b1a55cb 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeAwareReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java @@ -6,35 +6,25 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; -import java.io.BufferedReader; -import java.io.IOException; -import java.io.Reader; - -import org.checkerframework.checker.nullness.qual.Nullable; - import net.sourceforge.pmd.internal.util.AssertionUtil; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.FragmentedDocBuilder; import net.sourceforge.pmd.util.document.TextDocument; /** - * A reader that may interpret escapes in its input text. It records - * where escapes occurred, and can translate an offset in the translated - * document (the "output") to an offset in the original input. - * The implementation is optimised for the case where there are few escapes. - * {@link CharStream} is the API to navigate on a translated document - * (with arbitrary backtrack abilities). + * An object that can translate an input document into an output document, + * typically by replacing escape sequences with the character they represent. * - *

This is useful to back a {@link CharStream} for JavaCC implementation, - * but can also be used as a plain {@link Reader} if using other parser/lexer - * implementations. The reader behaviour is optimised for block IO and has - * poor char-by-char performance. Use a {@link BufferedReader} if you need it. - * - *

The default implementation does not perform any escape translation. + *

This is an abstract class because the default implementation does not + * perform any escape processing. Subclasses refine this behavior. */ @SuppressWarnings("PMD.AssignmentInOperand") -public class EscapeAwareReader extends Reader { +public abstract class EscapeTranslator implements AutoCloseable { + // Note that this can easily be turned into a java.io.Reader with + // efficient block IO, optimized for the common case where there are + // few or no escapes. This is part of the history of this file, but + // was removed for simplicity. /** * Source characters. When there is an escape, eg \ u00a0, the @@ -51,47 +41,30 @@ public class EscapeAwareReader extends Reader { private Chars curEscape; private int offInEscape; - public EscapeAwareReader(Chars input) { - AssertionUtil.requireParamNotNull("input", input); - this.input = input; - bufpos = 0; - escapes = new FragmentedDocBuilder(input); + public EscapeTranslator(TextDocument original) { + AssertionUtil.requireParamNotNull("builder", original); + this.input = original.getText(); + this.bufpos = 0; + this.escapes = new FragmentedDocBuilder(original); } + /** - * Translate all the input in the buffer. This is fed to a cursor initialized to zero. + * Translate all the input in the buffer. */ - public TextDocument translate(TextDocument source) throws IOException, MalformedSourceException { - readUnchecked(null, 0, Integer.MAX_VALUE); - return escapes.build(source); - } - - - @Override - public int read(final char[] cbuf, final int off, int len) throws IOException, MalformedSourceException { - if (off < 0 || len < 0 || len + off > cbuf.length) { - throw new IndexOutOfBoundsException("cbuf len=" + cbuf.length + " off=" + off + " len=" + len); - } - return readUnchecked(cbuf, off, len); - } - - // if cbuf is null we just want to record escapes - private int readUnchecked(char @Nullable [] cbuf, int off, int len) throws IOException, MalformedSourceException { + public TextDocument translateDocument() throws MalformedSourceException { ensureOpen(); if (this.bufpos == input.length()) { - return -1; + return escapes.build(); } - len = min(len, input.length()); // remove Integer.MAX_VALUE + final int len = input.length(); // remove Integer.MAX_VALUE int readChars = 0; while (readChars < len && (this.bufpos < input.length() || curEscape != null)) { if (curEscape != null) { int toRead = min(len - readChars, curEscape.length() - offInEscape); - if (cbuf != null) { - curEscape.getChars(0, cbuf, off + readChars, toRead); - } readChars += toRead; offInEscape += toRead; @@ -109,17 +82,13 @@ public class EscapeAwareReader extends Reader { assert newlyReadChars >= 0 && (readChars + newlyReadChars) <= len; - if (newlyReadChars != 0) { - if (cbuf != null) { - input.getChars(bpos, cbuf, off + readChars, newlyReadChars); - } - } else if (nextJump == input.length()) { + if (newlyReadChars == 0 && nextJump == input.length()) { // eof break; } readChars += newlyReadChars; } - return readChars; + return escapes.build(); } /** @@ -142,26 +111,19 @@ public class EscapeAwareReader extends Reader { return startOffsetInclusive; } - @Override - public void close() throws IOException { + public void close() { this.bufpos = -1; this.input = null; } /** Check to make sure that the stream has not been closed */ - protected void ensureOpen() throws IOException { + protected void ensureOpen() { if (input == null) { - throw new IOException("Stream closed"); + throw new IllegalStateException("Closed"); } } - @Override - public boolean ready() throws IOException { - ensureOpen(); - return true; - } - /** * Returns the offset in the input text of the given translated offset. * This includes the length of any unicode escapes. @@ -175,7 +137,7 @@ public class EscapeAwareReader extends Reader { * inputOffset(2) = 7 // includes the length of the escape * */ - public int inputOffset(int outputOffset) { + protected int inputOffset(int outputOffset) { return escapes.inputOffsetAt(outputOffset); } @@ -186,14 +148,14 @@ public class EscapeAwareReader extends Reader { * inefficient but currently is only used for error messages (which * obviously are exceptional). */ - public int getLine(int idxInInput) { + protected int getLine(int idxInInput) { return StringUtil.lineNumberAt(input, idxInInput); } /** * @see #getLine(int) */ - public int getColumn(int idxInInput) { + protected int getColumn(int idxInInput) { return StringUtil.columnNumberAt(input, idxInInput); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java deleted file mode 100644 index 68516ad79c..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/FragmentedDocBuilder.java +++ /dev/null @@ -1,247 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc.io; - - -import java.io.IOException; - -import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; - -import net.sourceforge.pmd.lang.LanguageVersion; -import net.sourceforge.pmd.util.document.Chars; -import net.sourceforge.pmd.util.document.FileLocation; -import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.TextRegion; - -final class FragmentedDocBuilder { - - private final Chars mainBuf; - - private Fragment lastFragment; - private Fragment firstFragment; - - private int curOffInInput; - - FragmentedDocBuilder(Chars buffer) { - this.mainBuf = buffer; - } - - /** - * Add a new fragment. - * - * @param startInInput Start (inclusive) of the overwritten text in the source - * @param endInInput End (exclusive) ... - * @param translation Characters with which the range startInInput..endInInput are overwritten. - * This may be empty. - */ - void recordDelta(int startInInput, int endInInput, Chars translation) { - assert curOffInInput <= startInInput : "Already moved past " + curOffInInput + ", cannot add delta at " + startInInput; - assert startInInput <= endInInput : "Offsets must be ordered"; - assert translation != null : "Translation cannot be null"; - - int inLength = endInInput - startInInput; - if (firstFragment == null) { - assert lastFragment == null; - firstFragment = new Fragment(null, startInInput, mainBuf.slice(0, startInInput)); - lastFragment = new Fragment(firstFragment, inLength, translation); - curOffInInput = endInInput; - return; - } - - Fragment last = lastFragment; - int prevLen = startInInput - curOffInInput; - if (prevLen != 0) { - last = new Fragment(last, prevLen, mainBuf.slice(curOffInInput, prevLen)); - } - last = new Fragment(last, inLength, translation); - this.lastFragment = last; - this.curOffInInput = endInInput; - } - - TextDocument build(TextDocument original) { - if (firstFragment == null) { - // No deltas in whole document, there's a single fragment - // This is the case for > 97% of Java files (source: OpenJDK) - return original; - } else { - if (curOffInInput < mainBuf.length()) { - // there's some text left between the last fragment and the end of the doc - int remLen = mainBuf.length() - curOffInInput; - Chars remainder = mainBuf.slice(curOffInInput, remLen); - lastFragment = new Fragment(lastFragment, remLen, remainder); - } - return new FragmentedTextDocument(original, firstFragment, lastFragment); - } - } - - int inputOffsetAt(int outputOffset) { - return inputOffsetAt(outputOffset, firstFragment); - } - - static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment) { - Fragment f = firstFragment; - if (f == null) { - return outputOffset; - } - while (f.next != null && f.inEnd() < outputOffset) { - f = f.next; - } - return f.outToIn(outputOffset); - } - - static final class FragmentedTextDocument implements TextDocument { - - private final Fragment firstFragment; - private final Chars text; - private final TextDocument base; - - FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) { - assert firstFragment != lastFragment; // NOPMD - this.firstFragment = firstFragment; - this.text = toChars(firstFragment, lastFragment); - this.base = base; - } - - private static Chars toChars(Fragment firstFragment, Fragment lastFragment) { - StringBuilder sb = new StringBuilder(lastFragment.outEnd()); - Fragment f = firstFragment; - while (f != null) { - f.getChars().appendChars(sb); - f = f.next; - } - return Chars.wrap(sb); - } - - @Override - public int translateOffset(int outputOffset) { - // todo this would be pretty slow when we're in the middle of some escapes - // we could check save the fragment last accessed to speed it up, and look forwards & backwards - return base.translateOffset(inputOffsetAt(outputOffset, firstFragment)); - } - - @Override - public Chars getText() { - return text; - } - - @Override - public long getChecksum() { - return base.getChecksum(); - } - - @Override - public LanguageVersion getLanguageVersion() { - return base.getLanguageVersion(); - } - - @Override - public String getPathId() { - return base.getPathId(); - } - - @Override - public String getDisplayName() { - return base.getDisplayName(); - } - - @Override - public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { - return base.createLineRange(startLineInclusive, endLineInclusive); - } - - @Override - public Chars sliceOriginalText(TextRegion region) { - return base.sliceOriginalText(translateRegion(region)); - } - - @Override - public FileLocation toLocation(TextRegion region) { - return base.toLocation(translateRegion(region)); - } - - @Override - public @NonNull TextRegion translateRegion(TextRegion region) { - return TextRegion.fromBothOffsets(translateOffset(region.getStartOffset()), - translateOffset(region.getEndOffset())); - } - - @Override - public void close() throws IOException { - base.close(); - } - } - - /** - * A delta from the original text to the translated text. This maps - * a region of the original document to some new characters. - */ - static final class Fragment { - - private final Chars chars; - - final @Nullable Fragment prev; - @Nullable Fragment next; - - private final int inLength; - private final int outStart; - private final int inStart; - - Fragment(@Nullable Fragment prev, int inLength, Chars chars) { - this.chars = chars; - this.prev = prev; - this.inLength = inLength; - if (prev != null) { - prev.next = this; - this.outStart = prev.outEnd(); - this.inStart = prev.inEnd(); - } else { - this.outStart = 0; - this.inStart = 0; - } - } - - public Chars getChars() { - return chars; - } - - int outStart() { - return outStart; - } - - int outLen() { - return chars.length(); - } - - int outEnd() { - return outStart() + outLen(); - } - - int inStart() { - return inStart; - } - - int inLen() { - return inLength; - } - - int inEnd() { - return inStart() + inLen(); - } - - int outToIn(int outOffset) { - return inStart() + (outOffset - outStart()); - } - - boolean contains(int outOffset) { - return outStart() <= outOffset && outEnd() > outOffset; - } - - @Override - public String toString() { - return "Fragment[" + inStart() + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; - } - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java similarity index 91% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java index 7ccea7bf0c..17c55043de 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReader.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java @@ -5,15 +5,16 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; /** - * An implementation of {@link EscapeAwareReader} that translates Java + * An implementation of {@link EscapeTranslator} that translates Java * unicode escapes. */ @SuppressWarnings("PMD.AssignmentInOperand") -public final class JavaEscapeReader extends BackslashEscapeReader { +public final class JavaEscapeTranslator extends BackslashEscapeTranslator { - public JavaEscapeReader(Chars input) { + public JavaEscapeTranslator(TextDocument input) { super(input); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java index 32ddd0a4e3..9b60c69724 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java @@ -109,6 +109,14 @@ public final class Chars implements CharSequence { str.getChars(start, start + count, cbuf, dstBegin); } + public void getChars(int srcBegin, CharBuffer buffer, int count) { + if (count == 0) { + return; + } + int start = idx(srcBegin); + str.getChars(start, start + count, cbuf, dstBegin); + } + /** * Appends the character range identified by offset and length into * the string builder. This is much more efficient than calling diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java new file mode 100644 index 0000000000..aec408bef1 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java @@ -0,0 +1,81 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.document; + + +import net.sourceforge.pmd.util.document.FragmentedTextDocument.Fragment; + +public final class FragmentedDocBuilder { + + private final Chars mainBuf; + private final TextDocument original; + + private Fragment lastFragment; + private Fragment firstFragment; + + private int curOffInInput; + + public FragmentedDocBuilder(TextDocument original) { + this.mainBuf = original.getText(); + this.original = original; + } + + public Chars inputChars() { + return mainBuf; + } + + /** + * Add a new fragment. + * + * @param startInInput Start (inclusive) of the overwritten text in the source + * @param endInInput End (exclusive) ... + * @param translation Characters with which the range startInInput..endInInput are overwritten. + * This may be empty. + */ + public void recordDelta(int startInInput, int endInInput, Chars translation) { + assert curOffInInput <= startInInput : "Already moved past " + curOffInInput + ", cannot add delta at " + startInInput; + assert startInInput <= endInInput : "Offsets must be ordered"; + assert translation != null : "Translation cannot be null"; + + int inLength = endInInput - startInInput; + if (firstFragment == null) { + assert lastFragment == null; + firstFragment = new Fragment(null, startInInput, mainBuf.slice(0, startInInput)); + lastFragment = new Fragment(firstFragment, inLength, translation); + curOffInInput = endInInput; + return; + } + + Fragment last = lastFragment; + int prevLen = startInInput - curOffInInput; + if (prevLen != 0) { + last = new Fragment(last, prevLen, mainBuf.slice(curOffInInput, prevLen)); + } + last = new Fragment(last, inLength, translation); + this.lastFragment = last; + this.curOffInInput = endInInput; + } + + public TextDocument build() { + if (firstFragment == null) { + // No deltas in whole document, there's a single fragment + // This is the case for > 97% of Java files (source: OpenJDK) + return original; + } else { + if (curOffInInput < mainBuf.length()) { + // there's some text left between the last fragment and the end of the doc + int remLen = mainBuf.length() - curOffInInput; + Chars remainder = mainBuf.slice(curOffInInput, remLen); + lastFragment = new Fragment(lastFragment, remLen, remainder); + } + return new FragmentedTextDocument(original, firstFragment, lastFragment); + } + } + + public int inputOffsetAt(int outputOffset) { + return FragmentedTextDocument.inputOffsetAt(outputOffset, firstFragment); + } + +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java new file mode 100644 index 0000000000..add5706d51 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java @@ -0,0 +1,180 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.document; + +import java.io.IOException; + +import org.checkerframework.checker.nullness.qual.NonNull; +import org.checkerframework.checker.nullness.qual.Nullable; + +import net.sourceforge.pmd.lang.LanguageVersion; + +/** + * A text document built as a set of deltas over another document. + */ +final class FragmentedTextDocument implements TextDocument { + + private final Fragment firstFragment; + private final Chars text; + private final TextDocument base; + + FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) { + assert firstFragment != lastFragment; // NOPMD + this.firstFragment = firstFragment; + this.text = toChars(firstFragment, lastFragment); + this.base = base; + } + + private static Chars toChars(Fragment firstFragment, Fragment lastFragment) { + StringBuilder sb = new StringBuilder(lastFragment.outEnd()); + Fragment f = firstFragment; + while (f != null) { + f.getChars().appendChars(sb); + f = f.next; + } + return Chars.wrap(sb); + } + + @Override + public int inputOffset(int outputOffset) { + // todo this would be pretty slow when we're in the middle of some escapes + // we could check save the fragment last accessed to speed it up, and look forwards & backwards + return base.inputOffset(inputOffsetAt(outputOffset, firstFragment)); + } + + @Override + public Chars getText() { + return text; + } + + @Override + public long getChecksum() { + return base.getChecksum(); + } + + @Override + public LanguageVersion getLanguageVersion() { + return base.getLanguageVersion(); + } + + @Override + public String getPathId() { + return base.getPathId(); + } + + @Override + public String getDisplayName() { + return base.getDisplayName(); + } + + @Override + public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { + return base.createLineRange(startLineInclusive, endLineInclusive); + } + + @Override + public Chars sliceOriginalText(TextRegion region) { + return base.sliceOriginalText(inputRegion(region)); + } + + @Override + public FileLocation toLocation(TextRegion region) { + return base.toLocation(inputRegion(region)); + } + + @Override + public @NonNull TextRegion inputRegion(TextRegion outputRegion) { + return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset()), + inputOffset(outputRegion.getEndOffset())); + } + + @Override + public void close() throws IOException { + base.close(); + } + + static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment) { + Fragment f = firstFragment; + if (f == null) { + return outputOffset; + } + while (f.next != null && f.inEnd() < outputOffset) { + f = f.next; + } + return f.outToIn(outputOffset); + } + + + /** + * A delta from the original text to the translated text. This maps + * a region of the original document to some new characters. + */ + static final class Fragment { + + private final Chars chars; + + final @Nullable Fragment prev; + @Nullable Fragment next; + + private final int inLength; + private final int outStart; + private final int inStart; + + Fragment(@Nullable Fragment prev, int inLength, Chars chars) { + this.chars = chars; + this.prev = prev; + this.inLength = inLength; + if (prev != null) { + prev.next = this; + this.outStart = prev.outEnd(); + this.inStart = prev.inEnd(); + } else { + this.outStart = 0; + this.inStart = 0; + } + } + + public Chars getChars() { + return chars; + } + + int outStart() { + return outStart; + } + + int outLen() { + return chars.length(); + } + + int outEnd() { + return outStart() + outLen(); + } + + int inStart() { + return inStart; + } + + int inLen() { + return inLength; + } + + int inEnd() { + return inStart() + inLen(); + } + + int outToIn(int outOffset) { + return inStart() + (outOffset - outStart()); + } + + boolean contains(int outOffset) { + return outStart() <= outOffset && outEnd() > outOffset; + } + + @Override + public String toString() { + return "Fragment[" + inStart() + ".." + inEnd() + " -> " + outStart() + ".." + outEnd() + "]" + chars; + } + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index a48a62ad47..42d123b3d4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -112,13 +112,13 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { } @Override - public int translateOffset(int outOffset) { + public int inputOffset(int outOffset) { return outOffset; } @Override - public TextRegion translateRegion(TextRegion region) { - return region; + public TextRegion inputRegion(TextRegion outputRegion) { + return outputRegion; } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index b0ecdf7344..591a0fd51e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -81,26 +81,36 @@ public interface TextDocument extends Closeable { /** * Returns the input offset for the given output offset. This maps * back an offset in the coordinate system of this document, to the - * coordinate system of the original document. + * coordinate system of the original document. This includes the + * length of any unicode escapes. + * + * 

+     * input:      "a\u00a0b"   (original document)
+     * translated: "a b"        (this document)
+     *
+     * translateOffset(0) = 0
+     * translateOffset(1) = 1
+     * translateOffset(2) = 7 // includes the length of the escape
+     *
* * @param outOffset Output offset * * @return Input offset */ - int translateOffset(int outOffset); + int inputOffset(int outOffset); /** * Translate a region given in the the coordinate system of this * document, to the coordinate system of the original document. * This works as if creating a new region with both start and end - * offsets translated through {@link #translateOffset(int)}. The + * offsets translated through {@link #inputOffset(int)}. The * returned region may have a different length. * - * @param region Output region + * @param outputRegion Output region * * @return Input region */ - TextRegion translateRegion(TextRegion region); + TextRegion inputRegion(TextRegion outputRegion); /** diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index f26f6fd7d1..ca032d6f82 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -18,7 +18,6 @@ import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; public class CharStreamImplTest { @@ -139,15 +138,18 @@ public class CharStreamImplTest { stream.readChar(); } - public CharStream simpleCharStream(String abcd) throws IOException { + public CharStream simpleCharStream(String abcd) { return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion))); } - public CharStream javaCharStream(String abcd) throws IOException { + public CharStream javaCharStream(String abcd) { return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion)) { + @Override - public EscapeAwareReader newReader(Chars text) { - return new JavaEscapeReader(text); + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { + return translator.translateDocument(); + } } }); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index 28a90609dc..2db00f2546 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -16,8 +16,8 @@ import net.sourceforge.pmd.util.document.Chars; public class JavaEscapeReaderTest { @NonNull - public JavaEscapeReader readString(String input) { - return new JavaEscapeReader(Chars.wrap(input)); + public JavaEscapeTranslator readString(String input) { + return new JavaEscapeTranslator(Chars.wrap(input)); } @@ -25,7 +25,7 @@ public class JavaEscapeReaderTest { public void testSimpleRead() throws IOException { String input = "abcdede"; - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[12]; @@ -40,7 +40,7 @@ public class JavaEscapeReaderTest { public void testNotAnEscape1Read() throws IOException { String input = "abc\\dede"; - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[12]; @@ -55,7 +55,7 @@ public class JavaEscapeReaderTest { public void testNotAnEscape1Read2() throws IOException { String input = "abc\\\\\\dede"; - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[12]; @@ -71,7 +71,7 @@ public class JavaEscapeReaderTest { String input = "abc\\\\\\dede"; // ^ - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[12]; @@ -96,7 +96,7 @@ public class JavaEscapeReaderTest { public void testAnEscapeStopAtEnd() throws IOException { String input = "abc\\\\\\u00a0dede"; - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[12]; @@ -116,7 +116,7 @@ public class JavaEscapeReaderTest { public void testSeveralEscapes() throws IOException { String input = "abc\\\\\\u00a0d\\uu00a0ede"; - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[20]; @@ -136,7 +136,7 @@ public class JavaEscapeReaderTest { public void testAnEscapeInsideBlock() throws IOException { String input = "abc\\\\\\u00a0dede\\u00a0"; - try (JavaEscapeReader r = readString(input)) { + try (JavaEscapeTranslator r = readString(input)) { char[] chars = new char[12]; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index f824c5f5cd..c3f6830508 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -11,11 +11,11 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.cpp.ast.CppEscapeReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.cpp.ast.CppEscapeTranslator; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -103,9 +103,12 @@ public class CPPTokenizer extends JavaCCTokenizer { protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { textDoc = TextDocument.readOnlyString(maybeSkipBlocks(textDoc.getText()), textDoc.getDisplayName(), textDoc.getLanguageVersion()); return new JavaccTokenDocument(textDoc) { + @Override - public EscapeAwareReader newReader(Chars text) { - return new CppEscapeReader(text); + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + try (CppEscapeTranslator translator = new CppEscapeTranslator(text)) { + return translator.translateDocument(); + } } @Override diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java similarity index 81% rename from pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java rename to pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java index 86f247604d..10ffd18503 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeReader.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java @@ -4,15 +4,16 @@ package net.sourceforge.pmd.lang.cpp.ast; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeTranslator; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; -public class CppEscapeReader extends BackslashEscapeReader { +public class CppEscapeTranslator extends BackslashEscapeTranslator { private static final char NEWLINE = '\n'; private static final char CARRIAGE_RETURN = '\r'; - public CppEscapeReader(Chars input) { + public CppEscapeTranslator(TextDocument input) { super(input); } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 159a5ee243..d0422442ea 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -14,12 +14,11 @@ import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.SINGLE_LINE_COMME import org.checkerframework.checker.nullness.qual.Nullable; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; -import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -47,8 +46,10 @@ final class JavaTokenDocument extends JavaccTokenDocument { @Override - public EscapeAwareReader newReader(Chars text) { - return new JavaEscapeReader(text); + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { + return translator.translateDocument(); + } } @Override diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java index eced79d5ad..20cbf36803 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java @@ -85,6 +85,7 @@ public class ParserCornersTest extends BaseJavaTreeDumpTest { @Test public void testUnicodeEscapes() { + // todo i'd like to test the coordinates of the literals, but this has to wait for java-grammar to be merged java8.parse("public class Foo { String[] s = { \"Ven\\u00E4j\\u00E4\" }; }"); } diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index e160b67b19..f2c5f99e1c 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -9,10 +9,9 @@ import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; -import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; public class JSPTokenizer extends JavaCCTokenizer { @@ -26,8 +25,10 @@ public class JSPTokenizer extends JavaCCTokenizer { protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { return new JavaccTokenDocument(textDoc) { @Override - public EscapeAwareReader newReader(Chars text) { - return new JavaEscapeReader(text); + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { + return translator.translateDocument(); + } } }; } diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 56a6772478..95915d6cd0 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -7,12 +7,11 @@ package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeAwareReader; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeReader; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; -import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; /** @@ -28,9 +27,12 @@ public class VfTokenizer extends JavaCCTokenizer { @Override protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { return new JavaccTokenDocument(textDoc) { + @Override - public EscapeAwareReader newReader(Chars text) { - return new JavaEscapeReader(text); + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { + return translator.translateDocument(); + } } }; } From 3ee32effdaccdf2f0cf16dd200091b8b6da6c4d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 08:41:23 +0200 Subject: [PATCH 045/136] Refactor token documents Isolate static parts --- javacc-wrapper.xml | 13 +- .../pmd/cpd/internal/JavaCCTokenizer.java | 9 +- .../pmd/lang/ast/impl/javacc/CharStream.java | 10 +- .../ast/impl/javacc/JavaccTokenDocument.java | 124 +++++++++++++++++- .../ast/impl/javacc/JjtreeParserAdapter.java | 7 +- .../ast/impl/javacc/io/EscapeTranslator.java | 11 ++ .../sourceforge/pmd/util/document/Chars.java | 8 -- .../impl/javacc/io/CharStreamImplTest.java | 16 +-- .../impl/javacc/io/JavaEscapeReaderTest.java | 5 +- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 58 ++------ .../sourceforge/pmd/cpd/CppBlockSkipper.java | 40 ++++++ .../cpp/ast => cpd}/CppEscapeTranslator.java | 2 +- .../sourceforge/pmd/cpd/JavaTokenizer.java | 4 +- .../pmd/lang/java/ast/InternalApiBridge.java | 4 +- .../pmd/lang/java/ast/JavaParser.java | 7 +- .../pmd/lang/java/ast/JavaTokenDocument.java | 20 +-- .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 12 +- .../pmd/lang/jsp/ast/JspParser.java | 23 ++-- .../pmd/lang/modelica/ast/ModelicaParser.java | 11 +- .../modelica/ast/ModelicaTokenDocument.java | 23 ---- .../pmd/lang/plsql/ast/PLSQLParser.java | 13 +- .../sourceforge/pmd/cpd/PythonTokenizer.java | 20 +-- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 17 +-- .../sourceforge/pmd/lang/vf/ast/VfParser.java | 18 +-- .../sourceforge/pmd/lang/vm/ast/VmParser.java | 56 ++++---- 25 files changed, 296 insertions(+), 235 deletions(-) create mode 100644 pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java rename pmd-cpp/src/main/java/net/sourceforge/pmd/{lang/cpp/ast => cpd}/CppEscapeTranslator.java (96%) delete mode 100644 pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java diff --git a/javacc-wrapper.xml b/javacc-wrapper.xml index 60d4d70158..bebe61eb3f 100644 --- a/javacc-wrapper.xml +++ b/javacc-wrapper.xml @@ -60,7 +60,7 @@ - + @@ -416,6 +416,17 @@ public final class ${token-constants-name} \{${line.separator} + + }; + TOKEN_NAMES = java.util.Collections.unmodifiableList(java.util.Arrays.asList(tokenImage)); +]]> + + + diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index f4711d20a6..6cbbad4133 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -16,7 +16,7 @@ import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.FileAnalysisException; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.CpdCompat; import net.sourceforge.pmd.util.document.TextDocument; @@ -26,12 +26,11 @@ public abstract class JavaCCTokenizer implements Tokenizer { @SuppressWarnings("PMD.CloseResource") protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException, MalformedSourceException { TextDocument textDocument = TextDocument.create(CpdCompat.cpdCompat(sourceCode)); - JavaccTokenDocument tokenDoc = newTokenDoc(textDocument); - return makeLexerImpl(CharStream.create(tokenDoc)); + return makeLexerImpl(CharStream.create(textDocument, newTokenDoc())); } - protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { - return new JavaccTokenDocument(textDoc); + protected TokenDocumentBehavior newTokenDoc() { + return TokenDocumentBehavior.DEFAULT; } protected abstract TokenManager makeLexerImpl(CharStream sourceCode); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index da881ef53b..33aa6e0b16 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -7,6 +7,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.EOFException; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; @@ -35,11 +36,12 @@ public final class CharStream { } /** - * Create a new char stream for the given document. + * Create a new char stream for the given document. Note: this + * mutates the token document by translating its escapes. */ - public static CharStream create(JavaccTokenDocument doc) throws MalformedSourceException { - doc.doTranslate(); - return new CharStream(doc); + public static CharStream create(TextDocument doc, TokenDocumentBehavior behavior) throws MalformedSourceException { + TextDocument translated = behavior.translate(doc); + return new CharStream(new JavaccTokenDocument(translated, behavior)); } /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 67ccbf63d9..b37537ac7b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -4,10 +4,15 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.TextDocument; @@ -22,10 +27,121 @@ public class JavaccTokenDocument extends TokenDocument { private JavaccToken first; private TextDocument translatedDocument; - public JavaccTokenDocument(TextDocument textDocument) { + public JavaccTokenDocument(TextDocument textDocument, TokenDocumentBehavior behavior) { super(textDocument); } + public static class TokenDocumentBehavior { + + public static final TokenDocumentBehavior DEFAULT = new TokenDocumentBehavior(); + private final List tokenNames; + private final Function translator; + + private TokenDocumentBehavior() { + this(Collections.emptyList()); + } + + public TokenDocumentBehavior(List tokenNames) { + this(tokenNames, t -> t); + } + + public TokenDocumentBehavior(List tokenNames, + Function translator) { + this.tokenNames = tokenNames; + this.translator = translator; + } + + /** + * Returns true if the lexer should accumulate the image of MORE + * tokens into the StringBuilder jjimage. This is useless in our + * current implementations. The default returns false, which makes + * {@link CharStream#appendSuffix(StringBuilder, int)} a noop. + */ + public boolean useMarkSuffix() { + return false; + } + + /** + * Translate the escapes of the source document. The default implementation + * does not perform any escaping. + * + * @param text Source doc + * + * @see EscapeTranslator + */ + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + return text; + } + + + protected boolean isImagePooled(JavaccToken t) { + return false; + } + + /** + * Returns a string that describes the token kind. + * + * @param kind Kind of token + * + * @return A descriptive string + */ + public final @NonNull String describeKind(int kind) { + if (kind == JavaccToken.IMPLICIT_TOKEN) { + return ""; + } + String impl = describeKindImpl(kind); + if (impl != null) { + return impl; + } + return ""; + } + + /** + * Describe the given kind. If this returns a non-null value, then + * that's what {@link #describeKind(int)} will use. Otherwise a default + * implementation is used. + * + *

An implementation typically uses the JavaCC-generated array + * named {@code Constants.tokenImage}. Remember to + * check the bounds of the array. + * + * @param kind Kind of token + * + * @return A descriptive string, or null to use default + */ + protected @Nullable String describeKindImpl(int kind) { + if (kind >= 0 && kind < tokenNames.size()) { + return tokenNames.get(kind); + } + return null; + } + + + /** + * Creates a new token with the given kind. This is called back to + * by JavaCC-generated token managers (jjFillToken). Note that a + * created token is not guaranteed to end up in the final token chain. + * + * @param kind Kind of the token + * @param cs Char stream of the file. This can be used to get text + * coordinates and the image + * @param image Shared instance of the image token. If this is non-null, + * then no call to {@link CharStream#getTokenImage()} should be + * issued. + * + * @return A new token + */ + public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream cs, @Nullable String image) { + return new JavaccToken( + kind, + image == null ? cs.getTokenImageCs() : image, + cs.getStartOffset(), + cs.getEndOffset(), + self + ); + } + } + /** * Returns true if the lexer should accumulate the image of MORE * tokens into the StringBuilder jjimage. This is useless in our @@ -37,10 +153,12 @@ public class JavaccTokenDocument extends TokenDocument { } /** - * Create new (possibly) escaping reader for the given text. The default - * implementation doesn't do any escaping. + * Translate the escapes of the source document. The default implementation + * does not perform any escaping. * * @param text Source doc + * + * @see EscapeTranslator */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { return text; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java index 71a9da0a25..02cd2dafd2 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeParserAdapter.java @@ -8,7 +8,6 @@ import net.sourceforge.pmd.lang.Parser; import net.sourceforge.pmd.lang.ast.FileAnalysisException; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.RootNode; -import net.sourceforge.pmd.util.document.TextDocument; /** * Base implementation of the {@link Parser} interface for JavaCC language @@ -23,14 +22,12 @@ public abstract class JjtreeParserAdapter implements Parser // inheritance only } - protected abstract JavaccTokenDocument newDocumentImpl(TextDocument textDocument); + protected abstract JavaccTokenDocument.TokenDocumentBehavior tokenBehavior(); @Override public R parse(ParserTask task) throws ParseException { - JavaccTokenDocument doc = newDocumentImpl(task.getTextDocument()); - try { - CharStream charStream = CharStream.create(doc); + CharStream charStream = CharStream.create(task.getTextDocument(), tokenBehavior()); return parseImpl(charStream, task); } catch (FileAnalysisException tme) { throw tme.setFileName(task.getFileDisplayName()); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java index ee9b1a55cb..1d7f46e158 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java @@ -6,6 +6,8 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; +import java.util.function.Function; + import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; @@ -159,4 +161,13 @@ public abstract class EscapeTranslator implements AutoCloseable { return StringUtil.columnNumberAt(input, idxInInput); } + + public static Function translatorFor(Function translatorMaker) { + return original -> { + try (EscapeTranslator translator = translatorMaker.apply(original)) { + return translator.translateDocument(); + } + }; + } + } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java index 9b60c69724..32ddd0a4e3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/Chars.java @@ -109,14 +109,6 @@ public final class Chars implements CharSequence { str.getChars(start, start + count, cbuf, dstBegin); } - public void getChars(int srcBegin, CharBuffer buffer, int count) { - if (count == 0) { - return; - } - int start = idx(srcBegin); - str.getChars(start, start + count, cbuf, dstBegin); - } - /** * Appends the character range identified by offset and length into * the string builder. This is much more efficient than calling diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index ca032d6f82..725383344e 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -9,6 +9,7 @@ import static org.junit.Assert.fail; import java.io.EOFException; import java.io.IOException; +import java.util.Collections; import org.junit.Rule; import org.junit.Test; @@ -17,7 +18,7 @@ import org.junit.rules.ExpectedException; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.util.document.TextDocument; public class CharStreamImplTest { @@ -139,19 +140,12 @@ public class CharStreamImplTest { } public CharStream simpleCharStream(String abcd) { - return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion))); + return CharStream.create(TextDocument.readOnlyString(abcd, dummyVersion), TokenDocumentBehavior.DEFAULT); } public CharStream javaCharStream(String abcd) { - return CharStream.create(new JavaccTokenDocument(TextDocument.readOnlyString(abcd, dummyVersion)) { - - @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { - try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { - return translator.translateDocument(); - } - } - }); + return CharStream.create(TextDocument.readOnlyString(abcd, dummyVersion), + new TokenDocumentBehavior(Collections.emptyList(), EscapeTranslator.translatorFor(JavaEscapeTranslator::new))); } @Test diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index 2db00f2546..b6922c8a23 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -10,14 +10,17 @@ import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; +import net.sourceforge.pmd.lang.DummyLanguageModule; +import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; public class JavaEscapeReaderTest { @NonNull public JavaEscapeTranslator readString(String input) { - return new JavaEscapeTranslator(Chars.wrap(input)); + return new JavaEscapeTranslator(TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index c3f6830508..3905f23f3a 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -7,19 +7,14 @@ package net.sourceforge.pmd.cpd; import java.io.IOException; import java.util.Properties; -import org.checkerframework.checker.nullness.qual.Nullable; - import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; -import net.sourceforge.pmd.lang.cpp.ast.CppEscapeTranslator; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; -import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; -import net.sourceforge.pmd.util.document.TextFileContent; /** * The C++ tokenizer. @@ -62,59 +57,22 @@ public class CPPTokenizer extends JavaCCTokenizer { } } - /** - * @param chars Normalized chars - */ - private CharSequence maybeSkipBlocks(Chars chars) { - if (!skipBlocks) { - return chars; - } - - int i = 0; - int lastLineStart = 0; - boolean skip = false; - StringBuilder filtered = new StringBuilder(chars.length()); - while (i < chars.length()) { - if (chars.charAt(i) == TextFileContent.NORMALIZED_LINE_TERM_CHAR) { - Chars lastLine = chars.subSequence(lastLineStart, i); - Chars trimmed = lastLine.trim(); - if (trimmed.contentEquals(skipBlocksStart, true)) { - skip = true; - } else if (trimmed.contentEquals(skipBlocksEnd, true)) { - skip = false; - } - if (!skip) { - lastLine.appendChars(filtered); - } - // always add newline, to preserve line numbers - filtered.append(TextFileContent.NORMALIZED_LINE_TERM_CHAR); - lastLineStart = i + 1; - } - i++; - } - if (lastLineStart < i && !skip) { - chars.appendChars(filtered, lastLineStart, i - lastLineStart); - } - return filtered; - } - @Override - protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { - textDoc = TextDocument.readOnlyString(maybeSkipBlocks(textDoc.getText()), textDoc.getDisplayName(), textDoc.getLanguageVersion()); - return new JavaccTokenDocument(textDoc) { + protected TokenDocumentBehavior newTokenDoc() { + return new TokenDocumentBehavior(CppTokenKinds.TOKEN_NAMES) { @Override protected TextDocument translate(TextDocument text) throws MalformedSourceException { + if (skipBlocks) { + try (CppBlockSkipper translator = new CppBlockSkipper(text, skipBlocksStart, skipBlocksEnd)) { + text = translator.translateDocument(); + } + } try (CppEscapeTranslator translator = new CppEscapeTranslator(text)) { return translator.translateDocument(); } } - - @Override - protected @Nullable String describeKindImpl(int kind) { - return CppTokenKinds.describe(kind); - } }; } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java new file mode 100644 index 0000000000..694d5cd978 --- /dev/null +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java @@ -0,0 +1,40 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.cpd; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.TextDocument; + +/** + * + */ +class CppBlockSkipper extends EscapeTranslator { + + private final Pattern skipStart; + private final Pattern skipEnd; + + public CppBlockSkipper(TextDocument original, String skipStartMarker, String skipEndMarker) { + super(original); + skipStart = Pattern.compile("^" + Pattern.quote(skipStartMarker)); + skipEnd = Pattern.compile("^" + Pattern.quote(skipEndMarker)); + } + + @Override + protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException { + Matcher start = skipStart.matcher(input).region(this.bufpos, maxOff); + if (start.find()) { + Matcher end = skipStart.matcher(input).region(start.end(), maxOff); + if (end.find()) { + return recordEscape(start.start(), end.end(), Chars.EMPTY); + } + } + return super.gobbleMaxWithoutEscape(maxOff); + } +} diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java similarity index 96% rename from pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java rename to pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java index 10ffd18503..5a4f0257aa 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/lang/cpp/ast/CppEscapeTranslator.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.cpp.ast; +package net.sourceforge.pmd.cpd; import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeTranslator; import net.sourceforge.pmd.util.document.Chars; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index a9daf1c1e1..284bd05af5 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -44,8 +44,8 @@ public class JavaTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { - return InternalApiBridge.javaTokenDoc(textDoc); + protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() { + return InternalApiBridge.javaTokenDoc(); } @Override diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java index e8921d1ac1..840e06c235 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java @@ -182,7 +182,7 @@ public final class InternalApiBridge { ((AbstractAnyTypeDeclaration) declaration).setBinaryName(binaryName, canon); } - public static JavaccTokenDocument javaTokenDoc(TextDocument fullText) { - return new JavaTokenDocument(fullText); + public static JavaccTokenDocument.TokenDocumentBehavior javaTokenDoc() { + return JavaTokenDocument.INSTANCE; } } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index fc3e890532..8556ba4094 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -5,11 +5,10 @@ package net.sourceforge.pmd.lang.java.ast; import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker; -import net.sourceforge.pmd.util.document.TextDocument; /** * Adapter for the JavaParser, using the specified grammar version. @@ -27,8 +26,8 @@ public class JavaParser extends JjtreeParserAdapter { @Override - protected JavaccTokenDocument newDocumentImpl(TextDocument textDocument) { - return new JavaTokenDocument(textDocument); + protected TokenDocumentBehavior tokenBehavior() { + return JavaTokenDocument.INSTANCE; } @Override diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index d0422442ea..734da5a486 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -17,6 +17,7 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.TextDocument; @@ -24,12 +25,15 @@ import net.sourceforge.pmd.util.document.TextDocument; /** * {@link JavaccTokenDocument} for Java. */ -final class JavaTokenDocument extends JavaccTokenDocument { +final class JavaTokenDocument extends JavaccTokenDocument.TokenDocumentBehavior { - JavaTokenDocument(TextDocument fullText) { - super(fullText); + static final JavaTokenDocument INSTANCE = new JavaTokenDocument(); + + private JavaTokenDocument() { + super(JavaTokenKinds.TOKEN_NAMES, EscapeTranslator.translatorFor(JavaEscapeTranslator::new)); } + /** * Returns true if the given token is a Java comment. */ @@ -52,18 +56,14 @@ final class JavaTokenDocument extends JavaccTokenDocument { } } + @Override protected boolean isImagePooled(JavaccToken t) { return t.kind == IDENTIFIER; } @Override - protected @Nullable String describeKindImpl(int kind) { - return JavaTokenKinds.describe(kind); - } - - @Override - public JavaccToken createToken(int kind, CharStream jcs, @Nullable String image) { + public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream jcs, @Nullable String image) { switch (kind) { case RUNSIGNEDSHIFT: case RSIGNEDSHIFT: @@ -77,7 +77,7 @@ final class JavaTokenDocument extends JavaccTokenDocument { jcs.getTokenDocument() ); default: - return super.createToken(kind, jcs, image); + return super.createToken(self, kind, jcs, image); } } diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index f2c5f99e1c..73b4f1fc35 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -11,6 +11,7 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.jsp.ast.JspParser; import net.sourceforge.pmd.lang.jsp.ast.JspTokenKinds; import net.sourceforge.pmd.util.document.TextDocument; @@ -22,15 +23,8 @@ public class JSPTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { - return new JavaccTokenDocument(textDoc) { - @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { - try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { - return translator.translateDocument(); - } - } - }; + protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() { + return JspParser.getTokenBehavior(); } } diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java index 4252bf16d4..87cb1dcd6e 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/lang/jsp/ast/JspParser.java @@ -4,27 +4,22 @@ package net.sourceforge.pmd.lang.jsp.ast; -import org.checkerframework.checker.nullness.qual.Nullable; - +import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.util.document.TextDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; +import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; /** * JSP language parser. */ public final class JspParser extends JjtreeParserAdapter { + private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(JspTokenKinds.TOKEN_NAMES); + @Override - protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) { - return new JavaccTokenDocument(fullText) { - @Override - protected @Nullable String describeKindImpl(int kind) { - return JspTokenKinds.describe(kind); - } - }; + protected TokenDocumentBehavior tokenBehavior() { + return TOKEN_BEHAVIOR; } @Override @@ -32,4 +27,8 @@ public final class JspParser extends JjtreeParserAdapter { return new JspParserImpl(cs).CompilationUnit().addTaskInfo(task); } + @InternalApi + public static TokenDocumentBehavior getTokenBehavior() { + return TOKEN_BEHAVIOR; + } } diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java index 058941d176..c4163c0bf0 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java @@ -4,18 +4,21 @@ package net.sourceforge.pmd.lang.modelica.ast; +import org.checkerframework.checker.nullness.qual.Nullable; + import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.util.document.TextDocument; public class ModelicaParser extends JjtreeParserAdapter { + private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(ModelicaTokenKinds.TOKEN_NAMES); + @Override - protected JavaccTokenDocument newDocumentImpl(TextDocument textDocument) { - return new ModelicaTokenDocument(textDocument); + protected TokenDocumentBehavior tokenBehavior() { + return TOKEN_BEHAVIOR; } @Override diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java deleted file mode 100644 index 04afae82b5..0000000000 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaTokenDocument.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.modelica.ast; - -import org.checkerframework.checker.nullness.qual.Nullable; - -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.util.document.TextDocument; - - -public class ModelicaTokenDocument extends JavaccTokenDocument { - - public ModelicaTokenDocument(TextDocument textDocument) { - super(textDocument); - } - - @Override - protected @Nullable String describeKindImpl(int kind) { - return ModelicaTokenKinds.describe(kind); - } -} diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java index e2beed5425..6b7fc86d69 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java @@ -8,20 +8,17 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.ParseException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.util.document.TextDocument; public class PLSQLParser extends JjtreeParserAdapter { + private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(PLSQLTokenKinds.TOKEN_NAMES); + @Override - protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) { - return new JavaccTokenDocument(fullText) { - @Override - protected @Nullable String describeKindImpl(int kind) { - return PLSQLTokenKinds.describe(kind); - } - }; + protected TokenDocumentBehavior tokenBehavior() { + return TOKEN_BEHAVIOR; } @Override diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 70755980ea..bbd0f3c8bf 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -13,6 +13,7 @@ import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds; import net.sourceforge.pmd.util.document.TextDocument; @@ -23,27 +24,16 @@ public class PythonTokenizer extends JavaCCTokenizer { private static final Pattern STRING_NL_ESCAPE = Pattern.compile("\\\\\\r?\\n"); + private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(PythonTokenKinds.TOKEN_NAMES); + @Override protected TokenManager makeLexerImpl(CharStream sourceCode) { return PythonTokenKinds.newTokenManager(sourceCode); } @Override - protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { - return new PythonTokenDocument(textDoc); - } - - private static class PythonTokenDocument extends JavaccTokenDocument { - - PythonTokenDocument(TextDocument fullText) { - super(fullText); - } - - @Override - protected @Nullable String describeKindImpl(int kind) { - return PythonTokenKinds.describe(kind); - } - + protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() { + return TOKEN_BEHAVIOR; } @Override diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 95915d6cd0..dfe4665c6a 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -9,10 +9,10 @@ import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; -import net.sourceforge.pmd.util.document.TextDocument; /** * @author sergey.gorbaty @@ -25,16 +25,9 @@ public class VfTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument newTokenDoc(TextDocument textDoc) { - return new JavaccTokenDocument(textDoc) { - - @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { - try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { - return translator.translateDocument(); - } - } - }; + protected TokenDocumentBehavior newTokenDoc() { + return new JavaccTokenDocument.TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES, + EscapeTranslator.translatorFor(JavaEscapeTranslator::new)); } } diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java index b740cc8c63..07b02632dc 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/lang/vf/ast/VfParser.java @@ -4,27 +4,21 @@ package net.sourceforge.pmd.lang.vf.ast; -import org.checkerframework.checker.nullness.qual.Nullable; - -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.util.document.TextDocument; /** * Parser for the VisualForce language. */ public final class VfParser extends JjtreeParserAdapter { + private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES); + @Override - protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) { - return new JavaccTokenDocument(fullText) { - @Override - protected @Nullable String describeKindImpl(int kind) { - return VfTokenKinds.describe(kind); - } - }; + protected TokenDocumentBehavior tokenBehavior() { + return TOKEN_BEHAVIOR; } @Override diff --git a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java index 9826dc367c..97f3065fb1 100644 --- a/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java +++ b/pmd-vm/src/main/java/net/sourceforge/pmd/lang/vm/ast/VmParser.java @@ -7,20 +7,39 @@ package net.sourceforge.pmd.lang.vm.ast; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.ParseException; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; -import net.sourceforge.pmd.util.document.TextDocument; /** * Adapter for the VmParser. */ public class VmParser extends JjtreeParserAdapter { + private static final TokenDocumentBehavior TOKEN_BEHAVIOR = new TokenDocumentBehavior(VmTokenKinds.TOKEN_NAMES) { + + @Override + public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream cs, @Nullable String image) { + String realImage = image == null ? cs.getTokenImage() : image; + if (kind == VmTokenKinds.ESCAPE_DIRECTIVE) { + realImage = escapedDirective(realImage); + } + + return super.createToken(self, kind, cs, realImage); + } + + private String escapedDirective(String strImage) { + int iLast = strImage.lastIndexOf("\\"); + String strDirective = strImage.substring(iLast + 1); + return strImage.substring(0, iLast / 2) + strDirective; + } + }; + @Override - protected JavaccTokenDocument newDocumentImpl(TextDocument fullText) { - return new VmTokenDocument(fullText); + protected TokenDocumentBehavior tokenBehavior() { + return TOKEN_BEHAVIOR; } @Override @@ -29,33 +48,4 @@ public class VmParser extends JjtreeParserAdapter { } - private static class VmTokenDocument extends JavaccTokenDocument { - - VmTokenDocument(TextDocument fullText) { - super(fullText); - } - - @Override - protected @Nullable String describeKindImpl(int kind) { - return VmTokenKinds.describe(kind); - } - - @Override - public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { - String realImage = image == null ? cs.getTokenImage() : image; - if (kind == VmTokenKinds.ESCAPE_DIRECTIVE) { - realImage = escapedDirective(realImage); - } - - return super.createToken(kind, cs, realImage); - } - - private String escapedDirective(String strImage) { - int iLast = strImage.lastIndexOf("\\"); - String strDirective = strImage.substring(iLast + 1); - return strImage.substring(0, iLast / 2) + strDirective; - } - - } - } From e76a1d6eb8d32f620a567f76e4a8713a2d57a503 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 09:21:48 +0200 Subject: [PATCH 046/136] Aha! fix tests of c++ --- .../ast/impl/javacc/JavaccTokenDocument.java | 72 ++--------- .../util/document/FragmentedDocBuilder.java | 2 +- .../util/document/FragmentedTextDocument.java | 20 ++- .../pmd/util/document/RootTextDocument.java | 2 +- .../pmd/util/document/TextDocument.java | 6 +- .../impl/javacc/io/JavaEscapeReaderTest.java | 118 +++--------------- .../sourceforge/pmd/cpd/CppBlockSkipper.java | 10 +- .../cpd/testdata/continuation_intra_token.txt | 7 +- 8 files changed, 58 insertions(+), 179 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index b37537ac7b..201720662b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -22,13 +22,14 @@ import net.sourceforge.pmd.util.document.TextDocument; */ public class JavaccTokenDocument extends TokenDocument { + private final TokenDocumentBehavior behavior; final StringPool stringPool = new StringPool(); private JavaccToken first; - private TextDocument translatedDocument; public JavaccTokenDocument(TextDocument textDocument, TokenDocumentBehavior behavior) { super(textDocument); + this.behavior = behavior; } public static class TokenDocumentBehavior { @@ -70,7 +71,7 @@ public class JavaccTokenDocument extends TokenDocument { * @see EscapeTranslator */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { - return text; + return translator.apply(text); } @@ -149,7 +150,7 @@ public class JavaccTokenDocument extends TokenDocument { * {@link CharStream#appendSuffix(StringBuilder, int)} a noop. */ public boolean useMarkSuffix() { - return false; + return behavior.useMarkSuffix(); } /** @@ -161,19 +162,7 @@ public class JavaccTokenDocument extends TokenDocument { * @see EscapeTranslator */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { - return text; - } - - @Override - public TextDocument getTextDocument() { - return translatedDocument == null ? super.getTextDocument() - : translatedDocument; - } - - final void doTranslate() throws MalformedSourceException { - if (translatedDocument == null) { - translatedDocument = translate(super.getTextDocument()); - } + return behavior.translate(text); } /** @@ -213,7 +202,7 @@ public class JavaccTokenDocument extends TokenDocument { } protected boolean isImagePooled(JavaccToken t) { - return false; + return behavior.isImagePooled(t); } /** @@ -224,55 +213,14 @@ public class JavaccTokenDocument extends TokenDocument { * @return A descriptive string */ public final @NonNull String describeKind(int kind) { - if (kind == JavaccToken.IMPLICIT_TOKEN) { - return ""; - } - String impl = describeKindImpl(kind); - if (impl != null) { - return impl; - } - return ""; + return behavior.describeKind(kind); } /** - * Describe the given kind. If this returns a non-null value, then - * that's what {@link #describeKind(int)} will use. Otherwise a default - * implementation is used. - * - *

An implementation typically uses the JavaCC-generated array - * named {@code Constants.tokenImage}. Remember to - * check the bounds of the array. - * - * @param kind Kind of token - * - * @return A descriptive string, or null to use default - */ - protected @Nullable String describeKindImpl(int kind) { - return null; - } - - - /** - * Creates a new token with the given kind. This is called back to - * by JavaCC-generated token managers (jjFillToken). Note that a - * created token is not guaranteed to end up in the final token chain. - * - * @param kind Kind of the token - * @param cs Char stream of the file. This can be used to get text - * coordinates and the image - * @param image Shared instance of the image token. If this is non-null, - * then no call to {@link CharStream#getTokenImage()} should be - * issued. - * - * @return A new token + * @see TokenDocumentBehavior#createToken(JavaccTokenDocument, int, CharStream, String) */ public JavaccToken createToken(int kind, CharStream cs, @Nullable String image) { - return new JavaccToken( - kind, - image == null ? cs.getTokenImageCs() : image, - cs.getStartOffset(), - cs.getEndOffset(), - this - ); + return behavior.createToken(this, kind, cs, image); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java index aec408bef1..4813435bec 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java @@ -75,7 +75,7 @@ public final class FragmentedDocBuilder { } public int inputOffsetAt(int outputOffset) { - return FragmentedTextDocument.inputOffsetAt(outputOffset, firstFragment); + return FragmentedTextDocument.inputOffsetAt(outputOffset, firstFragment, true); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java index add5706d51..f04338996e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java @@ -38,10 +38,10 @@ final class FragmentedTextDocument implements TextDocument { } @Override - public int inputOffset(int outputOffset) { + public int inputOffset(int outputOffset, boolean inclusive) { // todo this would be pretty slow when we're in the middle of some escapes // we could check save the fragment last accessed to speed it up, and look forwards & backwards - return base.inputOffset(inputOffsetAt(outputOffset, firstFragment)); + return base.inputOffset(inputOffsetAt(outputOffset, firstFragment, inclusive), inclusive); } @Override @@ -86,8 +86,8 @@ final class FragmentedTextDocument implements TextDocument { @Override public @NonNull TextRegion inputRegion(TextRegion outputRegion) { - return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset()), - inputOffset(outputRegion.getEndOffset())); + return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true), + inputOffset(outputRegion.getEndOffset(), false)); } @Override @@ -95,14 +95,22 @@ final class FragmentedTextDocument implements TextDocument { base.close(); } - static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment) { + static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment, boolean inclusive) { Fragment f = firstFragment; if (f == null) { return outputOffset; } - while (f.next != null && f.inEnd() < outputOffset) { + while (f.next != null && f.outEnd() < outputOffset) { f = f.next; } + if (!inclusive && f.outEnd() == outputOffset) { + if (f.next != null) { + f = f.next; + // fallthrough + } else { + return f.outToIn(outputOffset) + 1; + } + } return f.outToIn(outputOffset); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index 42d123b3d4..aa1c10ba63 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -112,7 +112,7 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { } @Override - public int inputOffset(int outOffset) { + public int inputOffset(int outOffset, boolean inclusive) { return outOffset; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 591a0fd51e..f07a7870a1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -94,16 +94,18 @@ public interface TextDocument extends Closeable { * * * @param outOffset Output offset + * @param inclusive Whether the offset is to be interpreted as the index of a character (true), + * or the position after a character (false) * * @return Input offset */ - int inputOffset(int outOffset); + int inputOffset(int outOffset, boolean inclusive); /** * Translate a region given in the the coordinate system of this * document, to the coordinate system of the original document. * This works as if creating a new region with both start and end - * offsets translated through {@link #inputOffset(int)}. The + * offsets translated through {@link #inputOffset(int, boolean)}. The * returned region may have a different length. * * @param outputRegion Output region diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index b6922c8a23..a72f7e760c 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -6,11 +6,9 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import java.io.IOException; -import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Assert; import org.junit.Test; -import net.sourceforge.pmd.lang.DummyLanguageModule; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.TextDocument; @@ -18,9 +16,11 @@ import net.sourceforge.pmd.util.document.TextDocument; public class JavaEscapeReaderTest { - @NonNull - public JavaEscapeTranslator readString(String input) { - return new JavaEscapeTranslator(TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); + public TextDocument readString(String input) { + TextDocument intext = TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); + try (JavaEscapeTranslator translator = new JavaEscapeTranslator(intext)) { + return translator.translateDocument(); + } } @@ -28,14 +28,8 @@ public class JavaEscapeReaderTest { public void testSimpleRead() throws IOException { String input = "abcdede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, chars.length); - - Assert.assertEquals("Should have read the entire text", input.length(), read); - Assert.assertEquals(input, new String(chars, 0, input.length())); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap(input),r.getText()); } } @@ -43,14 +37,8 @@ public class JavaEscapeReaderTest { public void testNotAnEscape1Read() throws IOException { String input = "abc\\dede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, chars.length); - - Assert.assertEquals("Should have read the entire text", input.length(), read); - Assert.assertEquals(input, new String(chars, 0, input.length())); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap(input),r.getText()); } } @@ -58,40 +46,8 @@ public class JavaEscapeReaderTest { public void testNotAnEscape1Read2() throws IOException { String input = "abc\\\\\\dede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, chars.length); - - Assert.assertEquals("Should have read the entire text", input.length(), read); - Assert.assertEquals(input, new String(chars, 0, input.length())); - } - } - - @Test - public void testNotAnEscape1Read3SplitInTheMiddleOfBackslashes() throws IOException { - - String input = "abc\\\\\\dede"; - // ^ - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, 4); - - Assert.assertEquals("Should have read just enough", 4, read); - assertBufferIsJust("abc\\", chars, 0); - - read = r.read(chars, 4, 1); - - Assert.assertEquals(1, read); - assertBufferIsJust("abc\\\\", chars, 0); - - read = r.read(chars, 5, chars.length - 5); - - Assert.assertEquals(5, read); - assertBufferIsJust("abc\\\\\\dede", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap(input),r.getText()); } } @@ -99,19 +55,8 @@ public class JavaEscapeReaderTest { public void testAnEscapeStopAtEnd() throws IOException { String input = "abc\\\\\\u00a0dede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, 4); - - Assert.assertEquals(4, read); - assertBufferIsJust("abc\u00a0", chars, 0); - - read = r.read(chars, 4, 2); - - Assert.assertEquals(2, read); - assertBufferIsJust("abc\u00a0de", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap("abc\u00a0dede"), r.getText()); } } @@ -119,19 +64,8 @@ public class JavaEscapeReaderTest { public void testSeveralEscapes() throws IOException { String input = "abc\\\\\\u00a0d\\uu00a0ede"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[20]; - - int read = r.read(chars, 0, 5); - - Assert.assertEquals(5, read); - assertBufferIsJust("abc\u00a0d", chars, 0); - - read = r.read(chars, 5, 4); - - Assert.assertEquals(4, read); - assertBufferIsJust("abc\u00a0d\u00a0ede", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap("abc\u00a0d\u00a0ede"), r.getText()); } } @@ -139,26 +73,8 @@ public class JavaEscapeReaderTest { public void testAnEscapeInsideBlock() throws IOException { String input = "abc\\\\\\u00a0dede\\u00a0"; - try (JavaEscapeTranslator r = readString(input)) { - - char[] chars = new char[12]; - - int read = r.read(chars, 0, 12); - - Assert.assertEquals(9, read); - assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); - - read = r.read(chars, 9, chars.length - 9); - - Assert.assertEquals(-1, read); - assertBufferIsJust("abc\u00a0dede\u00a0", chars, 0); + try (TextDocument r = readString(input)) { + Assert.assertEquals(Chars.wrap("abc\u00a0dede\u00a0"), r.getText()); } } - - private static void assertBufferIsJust(String contents, char[] chars, int off) { - // asserts the rest of the buffer is null characters - char[] chars2 = new char[chars.length]; - contents.getChars(0, contents.length(), chars2, off); - Assert.assertArrayEquals(chars2, chars); - } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java index 694d5cd978..fa5be8ee91 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java @@ -18,19 +18,23 @@ import net.sourceforge.pmd.util.document.TextDocument; class CppBlockSkipper extends EscapeTranslator { private final Pattern skipStart; + private final String skipStartMarker; private final Pattern skipEnd; + private final String skipEndMarker; public CppBlockSkipper(TextDocument original, String skipStartMarker, String skipEndMarker) { super(original); - skipStart = Pattern.compile("^" + Pattern.quote(skipStartMarker)); - skipEnd = Pattern.compile("^" + Pattern.quote(skipEndMarker)); + skipStart = Pattern.compile("^(?i)" + Pattern.quote(skipStartMarker), Pattern.MULTILINE); + this.skipStartMarker = "\n" + skipStartMarker; + skipEnd = Pattern.compile("^(?i)" + Pattern.quote(skipEndMarker), Pattern.MULTILINE); + this.skipEndMarker = "\n" + skipEndMarker; } @Override protected int gobbleMaxWithoutEscape(int maxOff) throws MalformedSourceException { Matcher start = skipStart.matcher(input).region(this.bufpos, maxOff); if (start.find()) { - Matcher end = skipStart.matcher(input).region(start.end(), maxOff); + Matcher end = skipEnd.matcher(input).region(start.end(), maxOff); if (end.find()) { return recordEscape(start.start(), end.end(), Chars.EMPTY); } diff --git a/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt b/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt index abc147ce72..febb035697 100644 --- a/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt +++ b/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt @@ -1,10 +1,11 @@ [Image] or [Truncated image[ Bcol Ecol L1 - [void] 1 1 + [void] 1 2 L5 - [main] 2 1 + [main] 2 2 +L9 + [(] 2 2 L10 - [(] 1 2 [)] 2 2 L12 [{] 2 2 From c88919bbc70dd961509bebc6441fe391fae82d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 09:25:48 +0200 Subject: [PATCH 047/136] Cleanup --- .../pmd/cpd/internal/JavaCCTokenizer.java | 4 +- .../ast/impl/javacc/JavaccTokenDocument.java | 45 ++--- .../ast/impl/javacc/io/EscapeTranslator.java | 1 + .../impl/javacc/io/JavaEscapeReaderTest.java | 6 +- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 2 +- .../pmd/cpd/CppCharStreamTest.java | 2 +- .../sourceforge/pmd/cpd/JavaTokenizer.java | 2 +- .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 2 +- pmd-plsql/src/main/ant/alljavacc.xml | 179 ------------------ .../pmd/lang/plsql/ast/PLSQLParser.java | 5 +- .../sourceforge/pmd/cpd/PythonTokenizer.java | 2 +- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 2 +- 12 files changed, 24 insertions(+), 228 deletions(-) delete mode 100644 pmd-plsql/src/main/ant/alljavacc.xml diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 6cbbad4133..dc14dafa2f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -26,10 +26,10 @@ public abstract class JavaCCTokenizer implements Tokenizer { @SuppressWarnings("PMD.CloseResource") protected TokenManager getLexerForSource(SourceCode sourceCode) throws IOException, MalformedSourceException { TextDocument textDocument = TextDocument.create(CpdCompat.cpdCompat(sourceCode)); - return makeLexerImpl(CharStream.create(textDocument, newTokenDoc())); + return makeLexerImpl(CharStream.create(textDocument, tokenBehavior())); } - protected TokenDocumentBehavior newTokenDoc() { + protected TokenDocumentBehavior tokenBehavior() { return TokenDocumentBehavior.DEFAULT; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 201720662b..76be813888 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -11,6 +11,7 @@ import java.util.function.Function; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; +import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; @@ -18,12 +19,14 @@ import net.sourceforge.pmd.util.document.TextDocument; /** * Token document for Javacc implementations. This is a helper object - * for generated token managers. + * for generated token managers. Note: the extension point is a custom + * implementation of {@link TokenDocumentBehavior}, see {@link JjtreeParserAdapter#tokenBehavior()}, + * {@link JavaCCTokenizer#tokenBehavior()} */ -public class JavaccTokenDocument extends TokenDocument { +public final class JavaccTokenDocument extends TokenDocument { private final TokenDocumentBehavior behavior; - final StringPool stringPool = new StringPool(); + private final StringPool stringPool = new StringPool(); private JavaccToken first; @@ -143,28 +146,10 @@ public class JavaccTokenDocument extends TokenDocument { } } - /** - * Returns true if the lexer should accumulate the image of MORE - * tokens into the StringBuilder jjimage. This is useless in our - * current implementations. The default returns false, which makes - * {@link CharStream#appendSuffix(StringBuilder, int)} a noop. - */ - public boolean useMarkSuffix() { + boolean useMarkSuffix() { return behavior.useMarkSuffix(); } - /** - * Translate the escapes of the source document. The default implementation - * does not perform any escaping. - * - * @param text Source doc - * - * @see EscapeTranslator - */ - protected TextDocument translate(TextDocument text) throws MalformedSourceException { - return behavior.translate(text); - } - /** * Open the document. This is only meant to be used by a Javacc-generated * parser. @@ -193,26 +178,18 @@ public class JavaccTokenDocument extends TokenDocument { return first.next; } - final String computeImage(JavaccToken t) { + String computeImage(JavaccToken t) { CharSequence imageCs = t.getImageCs(); if (imageCs instanceof String) { return (String) imageCs; } - return stringPool.toString(imageCs, isImagePooled(t)); - } - - protected boolean isImagePooled(JavaccToken t) { - return behavior.isImagePooled(t); + return stringPool.toString(imageCs, behavior.isImagePooled(t)); } /** - * Returns a string that describes the token kind. - * - * @param kind Kind of token - * - * @return A descriptive string + * @see TokenDocumentBehavior#describeKind(int) */ - public final @NonNull String describeKind(int kind) { + public @NonNull String describeKind(int kind) { return behavior.describeKind(kind); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java index 1d7f46e158..90f929aa76 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java @@ -113,6 +113,7 @@ public abstract class EscapeTranslator implements AutoCloseable { return startOffsetInclusive; } + @Override public void close() { this.bufpos = -1; this.input = null; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index a72f7e760c..8c9b803484 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -29,7 +29,7 @@ public class JavaEscapeReaderTest { String input = "abcdede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap(input),r.getText()); + Assert.assertEquals(Chars.wrap(input), r.getText()); } } @@ -38,7 +38,7 @@ public class JavaEscapeReaderTest { String input = "abc\\dede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap(input),r.getText()); + Assert.assertEquals(Chars.wrap(input), r.getText()); } } @@ -47,7 +47,7 @@ public class JavaEscapeReaderTest { String input = "abc\\\\\\dede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap(input),r.getText()); + Assert.assertEquals(Chars.wrap(input), r.getText()); } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 3905f23f3a..fbb80a8f33 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -59,7 +59,7 @@ public class CPPTokenizer extends JavaCCTokenizer { @Override - protected TokenDocumentBehavior newTokenDoc() { + protected TokenDocumentBehavior tokenBehavior() { return new TokenDocumentBehavior(CppTokenKinds.TOKEN_NAMES) { @Override diff --git a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java index 50d3c3c409..0101ea3b67 100644 --- a/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java +++ b/pmd-cpp/src/test/java/net/sourceforge/pmd/cpd/CppCharStreamTest.java @@ -21,7 +21,7 @@ public class CppCharStreamTest { @NonNull public CharStream charStreamFor(String source) throws IOException { TextDocument textDoc = TextDocument.readOnlyString(source, TextFile.UNKNOWN_FILENAME, CpdCompat.dummyVersion()); - return CharStream.create(new CPPTokenizer().newTokenDoc(textDoc)); + return CharStream.create(textDoc, new CPPTokenizer().tokenBehavior()); } @Test diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index 284bd05af5..39d1eb3699 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -44,7 +44,7 @@ public class JavaTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() { + protected JavaccTokenDocument.TokenDocumentBehavior tokenBehavior() { return InternalApiBridge.javaTokenDoc(); } diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index 73b4f1fc35..c1dc624a56 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -23,7 +23,7 @@ public class JSPTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() { + protected JavaccTokenDocument.TokenDocumentBehavior tokenBehavior() { return JspParser.getTokenBehavior(); } diff --git a/pmd-plsql/src/main/ant/alljavacc.xml b/pmd-plsql/src/main/ant/alljavacc.xml deleted file mode 100644 index e33aef3418..0000000000 --- a/pmd-plsql/src/main/ant/alljavacc.xml +++ /dev/null @@ -1,179 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - public class - - - - - - public class Token implements java.io.Serializable - - - - - - public Token specialToken; - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java index 6b7fc86d69..a9ebee6e09 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/lang/plsql/ast/PLSQLParser.java @@ -4,13 +4,10 @@ package net.sourceforge.pmd.lang.plsql.ast; -import org.checkerframework.checker.nullness.qual.Nullable; - import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; public class PLSQLParser extends JjtreeParserAdapter { diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index bbd0f3c8bf..3ebcd5a7e1 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -32,7 +32,7 @@ public class PythonTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument.TokenDocumentBehavior newTokenDoc() { + protected JavaccTokenDocument.TokenDocumentBehavior tokenBehavior() { return TOKEN_BEHAVIOR; } diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index dfe4665c6a..8ec128ac17 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -25,7 +25,7 @@ public class VfTokenizer extends JavaCCTokenizer { } @Override - protected TokenDocumentBehavior newTokenDoc() { + protected TokenDocumentBehavior tokenBehavior() { return new JavaccTokenDocument.TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES, EscapeTranslator.translatorFor(JavaEscapeTranslator::new)); } From 16380af92d519267d33c41452583cdfb32f18fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 10:01:04 +0200 Subject: [PATCH 048/136] Clarify difference between getText/getTextRegion --- .../java/net/sourceforge/pmd/lang/ast/Node.java | 4 ++-- .../pmd/lang/ast/TextAvailableNode.java | 17 ++++++++++------- .../ast/impl/javacc/AbstractJjtreeNode.java | 4 ++-- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 2 +- .../pmd/lang/ast/impl/javacc/JjtreeNode.java | 7 ------- .../pmd/util/document/TextDocument.java | 16 ++++++++++++++++ 6 files changed, 31 insertions(+), 19 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java index f42b5e27a1..de9a1fc26e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java @@ -89,7 +89,7 @@ public interface Node extends Reportable { /** * Compare the coordinates of this node with the other one as if * with {@link FileLocation#COORDS_COMPARATOR}. The result is useless - * if both nodes are not from the same tree (todo check it?). + * if both nodes are not from the same tree. * * @param node Other node * @@ -102,7 +102,7 @@ public interface Node extends Reportable { } // Those are kept here because they're handled specially as XPath - // attributes + // attributes, for now @Override default int getBeginLine() { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java index 760161faca..0a7d0c85bf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java @@ -5,6 +5,7 @@ package net.sourceforge.pmd.lang.ast; import net.sourceforge.pmd.lang.rule.xpath.NoAttribute; +import net.sourceforge.pmd.util.document.TextDocument; import net.sourceforge.pmd.util.document.TextRegion; /** @@ -17,17 +18,19 @@ public interface TextAvailableNode extends Node { /** - * Returns the exact region of text delimiting - * the node in the underlying text document. Note - * that {@link #getReportLocation()} does not need - * to match this region. + * Returns the exact region of text delimiting the node in the + * underlying text document. Note that {@link #getReportLocation()} + * does not need to match this region. This region uses the translated + * coordinate system, ie the coordinate system of {@link #getTextDocument()}. */ TextRegion getTextRegion(); /** - * Returns the original source code underlying this node. In - * particular, for a {@link RootNode}, returns the whole text - * of the file. + * Returns the original source code underlying this node. In particular, + * for a {@link RootNode}, returns the whole text of the file. Note the + * difference between this method and {@code getTextDocument().getText().slice(getTextRegion())}. + * The latter is {@link TextDocument#sliceTranslatedText(TextRegion)}, + * the former (this method) is {@link TextDocument#sliceOriginalText(TextRegion)}. */ @NoAttribute CharSequence getText(); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java index a2a5035948..5d8e7d9873 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java @@ -50,7 +50,7 @@ public abstract class AbstractJjtreeNode, N e } @Override - public Chars getText() { + public final Chars getText() { return getTextDocument().sliceOriginalText(getTextRegion()); } @@ -60,7 +60,7 @@ public abstract class AbstractJjtreeNode, N e } @Override - public TextRegion getTextRegion() { + public final TextRegion getTextRegion() { return TextRegion.fromBothOffsets(getFirstToken().getStartOffset(), getLastToken().getEndOffset()); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index d4f433188b..2983cf1109 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -145,7 +145,7 @@ public class JavaccToken implements GenericToken { } @Override - public TextRegion getRegion() { + public final TextRegion getRegion() { return TextRegion.fromBothOffsets(startOffset, endOffset); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java index 741b77bbce..8d9ad2c53e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java @@ -8,7 +8,6 @@ import net.sourceforge.pmd.lang.ast.TextAvailableNode; import net.sourceforge.pmd.lang.ast.impl.GenericNode; import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.Reportable; -import net.sourceforge.pmd.util.document.TextRegion; /** * Base interface for nodes that are produced by a JJTree parser. Our @@ -22,12 +21,6 @@ public interface JjtreeNode> extends GenericNode, Tex @Override Chars getText(); - /** - * Returns the region delimiting the text of this node. - */ - @Override - TextRegion getTextRegion(); - JavaccToken getFirstToken(); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index f07a7870a1..21d5364a2b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -66,9 +66,25 @@ public interface TextDocument extends Closeable { * @param region A region, in the coordinate system of this document * * @return The slice of the original text that corresponds to the region + * + * @throws IndexOutOfBoundsException If the region is not a valid range */ Chars sliceOriginalText(TextRegion region); + /** + * Returns a slice of the source text. This is always equal to + * {@code getText().slice(region)}, as the text is the translated text. + * + * @param region A region, in the coordinate system of this document + * + * @return The slice of the original text that corresponds to the region + * + * @throws IndexOutOfBoundsException If the region is not a valid range + */ + default Chars sliceTranslatedText(TextRegion region) { + return getText().slice(region); + } + /** * Returns a checksum for the contents of the file. From 1ac7e43cacd669486dff1cec81926b8de611f6bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 10:31:00 +0200 Subject: [PATCH 049/136] Abstract some offset mapping logic This can be shared with subdocument views later --- .../pmd/util/document/BaseMappedDocument.java | 77 +++++++++++++++++++ .../util/document/FragmentedTextDocument.java | 51 +----------- .../pmd/util/document/RootTextDocument.java | 5 ++ .../pmd/util/document/TextDocument.java | 10 ++- 4 files changed, 94 insertions(+), 49 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/document/BaseMappedDocument.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/BaseMappedDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/BaseMappedDocument.java new file mode 100644 index 0000000000..8c422afa51 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/BaseMappedDocument.java @@ -0,0 +1,77 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.document; + +import java.io.IOException; + +import org.checkerframework.checker.nullness.qual.NonNull; + +/** + * Base class for documents that apply a transform to their output offsets. + * This includes translated documents, and slices (subdocument views). + */ +abstract class BaseMappedDocument implements TextDocument { + + protected final TextDocument base; + + BaseMappedDocument(TextDocument base) { + this.base = base; + } + + @Override + public long getChecksum() { + return base.getChecksum(); + } + + @Override + public String getPathId() { + return base.getPathId(); + } + + @Override + public String getDisplayName() { + return base.getDisplayName(); + } + + @Override + public Chars sliceOriginalText(TextRegion region) { + return base.sliceOriginalText(inputRegion(region)); + } + + @Override + public FileLocation toLocation(TextRegion region) { + return base.toLocation(inputRegion(region)); + } + + @Override + public @NonNull TextRegion inputRegion(TextRegion outputRegion) { + // note that inputOffset already recurses up to the original document, + // so that we don't have to call base.inputRegion on the produced region + return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true), + inputOffset(outputRegion.getEndOffset(), false)); + } + + @Override + public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { + // see the doc, lines do not need to be translated + return base.createLineRange(startLineInclusive, endLineInclusive); + } + + @Override + public int inputOffset(int outOffset, boolean inclusive) { + if (outOffset < 0 || outOffset > getLength()) { + throw new IndexOutOfBoundsException(); + } + return base.inputOffset(localOffsetTransform(outOffset, inclusive), inclusive); + } + + protected abstract int localOffsetTransform(int outOffset, boolean inclusive); + + + @Override + public void close() throws IOException { + base.close(); + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java index f04338996e..cbebe1556d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java @@ -4,9 +4,6 @@ package net.sourceforge.pmd.util.document; -import java.io.IOException; - -import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.LanguageVersion; @@ -14,17 +11,16 @@ import net.sourceforge.pmd.lang.LanguageVersion; /** * A text document built as a set of deltas over another document. */ -final class FragmentedTextDocument implements TextDocument { +final class FragmentedTextDocument extends BaseMappedDocument implements TextDocument { private final Fragment firstFragment; private final Chars text; - private final TextDocument base; FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) { + super(base); assert firstFragment != lastFragment; // NOPMD this.firstFragment = firstFragment; this.text = toChars(firstFragment, lastFragment); - this.base = base; } private static Chars toChars(Fragment firstFragment, Fragment lastFragment) { @@ -38,10 +34,10 @@ final class FragmentedTextDocument implements TextDocument { } @Override - public int inputOffset(int outputOffset, boolean inclusive) { + protected int localOffsetTransform(int outOffset, boolean inclusive) { // todo this would be pretty slow when we're in the middle of some escapes // we could check save the fragment last accessed to speed it up, and look forwards & backwards - return base.inputOffset(inputOffsetAt(outputOffset, firstFragment, inclusive), inclusive); + return inputOffsetAt(outOffset, firstFragment, inclusive); } @Override @@ -49,51 +45,12 @@ final class FragmentedTextDocument implements TextDocument { return text; } - @Override - public long getChecksum() { - return base.getChecksum(); - } @Override public LanguageVersion getLanguageVersion() { return base.getLanguageVersion(); } - @Override - public String getPathId() { - return base.getPathId(); - } - - @Override - public String getDisplayName() { - return base.getDisplayName(); - } - - @Override - public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { - return base.createLineRange(startLineInclusive, endLineInclusive); - } - - @Override - public Chars sliceOriginalText(TextRegion region) { - return base.sliceOriginalText(inputRegion(region)); - } - - @Override - public FileLocation toLocation(TextRegion region) { - return base.toLocation(inputRegion(region)); - } - - @Override - public @NonNull TextRegion inputRegion(TextRegion outputRegion) { - return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true), - inputOffset(outputRegion.getEndOffset(), false)); - } - - @Override - public void close() throws IOException { - base.close(); - } static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment, boolean inclusive) { Fragment f = firstFragment; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java index aa1c10ba63..4eca7d3f96 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/RootTextDocument.java @@ -128,6 +128,7 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { private static final String NOT_IN_RANGE = "Region [start=%d, end=%d[ is not in range of this document (length %d)"; private static final String INVALID_LINE_RANGE = "Line range %d..%d is not in range of this document (%d lines) (line numbers are 1-based)"; + private static final String INVALID_OFFSET = "Offset %d is not in range of this document (length %d) (line numbers are 0-based)"; static IndexOutOfBoundsException invalidLineRange(int start, int end, int numLines) { return new IndexOutOfBoundsException(String.format(INVALID_LINE_RANGE, start, end, numLines)); @@ -136,4 +137,8 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { static IndexOutOfBoundsException regionOutOfBounds(int start, int end, int maxLen) { return new IndexOutOfBoundsException(String.format(NOT_IN_RANGE, start, end, maxLen)); } + + static IndexOutOfBoundsException invalidOffset(int offset, int maxLen) { + return new IndexOutOfBoundsException(String.format(INVALID_OFFSET, offset, maxLen)); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 21d5364a2b..42f8e9c07d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -26,6 +26,8 @@ import net.sourceforge.pmd.util.datasource.DataSource; public interface TextDocument extends Closeable { // todo logical sub-documents, to support embedded languages // ideally, just slice the text, and share the positioner + // a problem with document slices becomes reference counting for the close routine + // todo text edition (there are some reverted commits in the branch // with part of this, including a lot of tests) @@ -143,11 +145,16 @@ public interface TextDocument extends Closeable { * Returns a region that spans the text of all the given lines. * This is intended to provide a replacement for {@link SourceCode#getSlice(int, int)}. * + *

Note that, as line numbers may only be obtained from {@link #toLocation(TextRegion)}, + * and hence are line numbers of the original source, both parameters + * must be line numbers of the source text and not the translated text + * that this represents. + * * @param startLineInclusive Inclusive start line number (1-based) * @param endLineInclusive Inclusive end line number (1-based) * * @throws IndexOutOfBoundsException If the arguments do not identify - * a valid region in this document + * a valid region in the source document */ TextRegion createLineRange(int startLineInclusive, int endLineInclusive); @@ -186,7 +193,6 @@ public interface TextDocument extends Closeable { @Override void close() throws IOException; - static TextDocument create(TextFile textFile) throws IOException { return new RootTextDocument(textFile); } From dd63a88e6a0c34826c14ee317250581119507d94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 2 Sep 2020 10:55:28 +0200 Subject: [PATCH 050/136] Cleanup cpp --- .../pmd/internal/util/AssertionUtil.java | 2 +- .../pmd/util/document/SourceCodePositioner.java | 2 +- .../java/net/sourceforge/pmd/cpd/CPPTokenizer.java | 11 ++++++----- .../net/sourceforge/pmd/cpd/CppBlockSkipper.java | 14 +++++++------- .../pmd/lang/java/ast/FormalComment.java | 4 +--- .../pmd/lang/java/ast/InternalApiBridge.java | 5 ----- 6 files changed, 16 insertions(+), 22 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/AssertionUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/AssertionUtil.java index 2f47421a24..4e3ad73af7 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/AssertionUtil.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/internal/util/AssertionUtil.java @@ -6,8 +6,8 @@ package net.sourceforge.pmd.internal.util; import java.util.Collection; -import java.util.regex.Pattern; import java.util.function.Function; +import java.util.regex.Pattern; import org.apache.commons.lang3.StringUtils; import org.checkerframework.checker.nullness.qual.NonNull; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/SourceCodePositioner.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/SourceCodePositioner.java index 4c008918cd..b3e1966208 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/SourceCodePositioner.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/SourceCodePositioner.java @@ -216,7 +216,7 @@ final class SourceCodePositioner { buf = new int[Math.max(1, bufSize)]; } - public Builder() { + Builder() { this(400); } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index fbb80a8f33..2dada00663 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -6,6 +6,7 @@ package net.sourceforge.pmd.cpd; import java.io.IOException; import java.util.Properties; +import java.util.regex.Pattern; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; @@ -22,8 +23,8 @@ import net.sourceforge.pmd.util.document.TextDocument; public class CPPTokenizer extends JavaCCTokenizer { private boolean skipBlocks; - private String skipBlocksStart; - private String skipBlocksEnd; + private Pattern skipBlocksStart; + private Pattern skipBlocksEnd; public CPPTokenizer() { setProperties(new Properties()); // set the defaults @@ -48,11 +49,11 @@ public class CPPTokenizer extends JavaCCTokenizer { if (skipBlocks) { String skipBlocksPattern = properties.getProperty(OPTION_SKIP_BLOCKS_PATTERN, DEFAULT_SKIP_BLOCKS_PATTERN); String[] split = skipBlocksPattern.split("\\|", 2); - skipBlocksStart = split[0]; + skipBlocksStart = CppBlockSkipper.compileSkipMarker(split[0]); if (split.length == 1) { - skipBlocksEnd = split[0]; + skipBlocksEnd = skipBlocksStart; } else { - skipBlocksEnd = split[1]; + skipBlocksEnd = CppBlockSkipper.compileSkipMarker(split[1]); } } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java index fa5be8ee91..4770579f60 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java @@ -18,16 +18,16 @@ import net.sourceforge.pmd.util.document.TextDocument; class CppBlockSkipper extends EscapeTranslator { private final Pattern skipStart; - private final String skipStartMarker; private final Pattern skipEnd; - private final String skipEndMarker; - public CppBlockSkipper(TextDocument original, String skipStartMarker, String skipEndMarker) { + static Pattern compileSkipMarker(String marker) { + return Pattern.compile("^(?i)" + Pattern.quote(marker), Pattern.MULTILINE); + } + + public CppBlockSkipper(TextDocument original, Pattern skipStartMarker, Pattern skipEndMarker) { super(original); - skipStart = Pattern.compile("^(?i)" + Pattern.quote(skipStartMarker), Pattern.MULTILINE); - this.skipStartMarker = "\n" + skipStartMarker; - skipEnd = Pattern.compile("^(?i)" + Pattern.quote(skipEndMarker), Pattern.MULTILINE); - this.skipEndMarker = "\n" + skipEndMarker; + skipStart = skipStartMarker; + skipEnd = skipEndMarker; } @Override diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java index c232b943c0..4f47dd5bbf 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/FormalComment.java @@ -18,9 +18,7 @@ public class FormalComment extends Comment { public FormalComment(JavaccToken t) { super(t); - -// findJavadocs(); - + findJavadocs(t); } @Override diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java index 840e06c235..d7706dae6c 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/InternalApiBridge.java @@ -26,7 +26,6 @@ import net.sourceforge.pmd.lang.java.types.JVariableSig.FieldSig; import net.sourceforge.pmd.lang.java.types.OverloadSelectionResult; import net.sourceforge.pmd.lang.java.types.internal.infer.TypeInferenceLogger; import net.sourceforge.pmd.lang.symboltable.Scope; -import net.sourceforge.pmd.util.document.TextDocument; /** * Acts as a bridge between outer parts of PMD and the restricted access @@ -96,10 +95,6 @@ public final class InternalApiBridge { return methodDeclaration; } - public static JavaccTokenDocument javaTokenDoc(String fullText) { - return new JavaTokenDocument(fullText); - } - public static void setSymbol(SymbolDeclaratorNode node, JElementSymbol symbol) { if (node instanceof ASTMethodDeclaration) { ((ASTMethodDeclaration) node).setSymbol((JMethodSymbol) symbol); From 8052d87727abd6c8900bd0ecabe4e361fb19ec53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 17 Sep 2020 23:07:46 +0200 Subject: [PATCH 051/136] Fix compil --- pmd-java/etc/grammar/Java.jjt | 2 +- .../pmd/lang/java/ast/ASTAnnotation.java | 2 +- .../pmd/lang/java/ast/TokenUtils.java | 33 ++++--------------- .../symboltable/MethodNameDeclaration.java | 2 +- 4 files changed, 9 insertions(+), 30 deletions(-) diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index ab9fe30349..b2eb1fbba4 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -240,7 +240,7 @@ import java.util.EnumSet; import java.util.List; import java.util.Set; import java.util.Map; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.GenericToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.Node; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTAnnotation.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTAnnotation.java index ce39a32c68..6ea17ad6bd 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTAnnotation.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTAnnotation.java @@ -50,7 +50,7 @@ public final class ASTAnnotation extends AbstractJavaTypeNode implements TypeNod @Deprecated @DeprecatedUntil700 public String getAnnotationName() { - return (String) getTypeNode().getText(); + return getTypeNode().getText().toString(); } /** diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/TokenUtils.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/TokenUtils.java index 7075123eca..c0d48d1617 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/TokenUtils.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/TokenUtils.java @@ -4,7 +4,6 @@ package net.sourceforge.pmd.lang.java.ast; -import java.util.Comparator; import java.util.Iterator; import java.util.NoSuchElementException; import java.util.Objects; @@ -22,37 +21,17 @@ final class TokenUtils { // mind: getBeginLine and getEndLine on JavaccToken are now very slow. - /** - * Assumes no two tokens overlap, and that the two tokens are from - * the same document. - */ - private static final Comparator TOKEN_POS_COMPARATOR - = Comparator.comparingInt(GenericToken::getStartInDocument); - private TokenUtils() { } - public static int compare(GenericToken t1, GenericToken t2) { - return TOKEN_POS_COMPARATOR.compare(t1, t2); - } - public static boolean isBefore(GenericToken t1, GenericToken t2) { - return t1.getStartInDocument() < t2.getStartInDocument(); - } - - public static boolean isAfter(GenericToken t1, GenericToken t2) { - return t1.getStartInDocument() > t2.getStartInDocument(); - - } - - - public static T nthFollower(T token, int n) { + public static > T nthFollower(T token, int n) { if (n < 0) { throw new IllegalArgumentException("Negative index?"); } while (n-- > 0 && token != null) { - token = (T) token.getNext(); + token = token.getNext(); } if (token == null) { throw new NoSuchElementException("No such token"); @@ -77,8 +56,8 @@ final class TokenUtils { * @throws NoSuchElementException If there's less than n tokens to the left of the anchor. */ // test only - public static T nthPrevious(T startHint, T anchor, int n) { - if (compare(startHint, anchor) >= 0) { + public static > T nthPrevious(T startHint, T anchor, int n) { + if (startHint.compareTo(anchor) >= 0) { throw new IllegalStateException("Wrong left hint, possibly not left enough"); } if (n <= 0) { @@ -88,12 +67,12 @@ final class TokenUtils { T target = startHint; T current = startHint; while (current != null && !current.equals(anchor)) { - current = (T) current.getNext(); + current = current.getNext(); // wait "n" iterations before starting to advance the target // then advance "target" at the same rate as "current", but // "n" tokens to the left if (numAway == n) { - target = (T) target.getNext(); + target = target.getNext(); } else { numAway++; } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/symboltable/MethodNameDeclaration.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/symboltable/MethodNameDeclaration.java index 984e0a46aa..65c3c54d7f 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/symboltable/MethodNameDeclaration.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/symboltable/MethodNameDeclaration.java @@ -116,6 +116,6 @@ public class MethodNameDeclaration extends AbstractNameDeclaration { @Override public String toString() { return "Method " + node.getImage() + ", line " + node.getBeginLine() + ", params = " - + ((ASTMethodDeclarator) node).getParameterCount(); + + getDeclarator().getArity(); } } From f6a57240c28ea1089d95235f468a25257c41b982 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 31 Oct 2020 01:10:45 +0100 Subject: [PATCH 052/136] Optimise fragmented doc builder offset access Just cache the most recently accessed fragment and assume we're working near it. The previous scheme was taking twice as much time as the rest of the run combined on a real run. --- .../ast/impl/javacc/io/EscapeTranslator.java | 23 +---------- .../util/document/FragmentedDocBuilder.java | 4 -- .../util/document/FragmentedTextDocument.java | 41 ++++++++++++++----- 3 files changed, 31 insertions(+), 37 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java index 90f929aa76..c403724920 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java @@ -128,28 +128,7 @@ public abstract class EscapeTranslator implements AutoCloseable { } /** - * Returns the offset in the input text of the given translated offset. - * This includes the length of any unicode escapes. - * - * 

-     * input:      "a\u00a0b"
-     * translated: "a b"
-     *
-     * inputOffset(0) = 0
-     * inputOffset(1) = 1
-     * inputOffset(2) = 7 // includes the length of the escape
-     *
- */ - protected int inputOffset(int outputOffset) { - return escapes.inputOffsetAt(outputOffset); - } - - /** - * The parameter is an *input* offset, if you got this offset from - * somewhere else than the input buffer you must first translate it - * back with {@link #inputOffset(int)}. This implementation is very - * inefficient but currently is only used for error messages (which - * obviously are exceptional). + * The parameter is an *input* offset. */ protected int getLine(int idxInInput) { return StringUtil.lineNumberAt(input, idxInInput); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java index 4813435bec..7c4397266d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java @@ -74,8 +74,4 @@ public final class FragmentedDocBuilder { } } - public int inputOffsetAt(int outputOffset) { - return FragmentedTextDocument.inputOffsetAt(outputOffset, firstFragment, true); - } - } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java index cbebe1556d..f497575a08 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedTextDocument.java @@ -16,11 +16,14 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc private final Fragment firstFragment; private final Chars text; + private Fragment lastAccessedFragment; + FragmentedTextDocument(TextDocument base, Fragment firstFragment, Fragment lastFragment) { super(base); assert firstFragment != lastFragment; // NOPMD this.firstFragment = firstFragment; this.text = toChars(firstFragment, lastFragment); + this.lastAccessedFragment = firstFragment; } private static Chars toChars(Fragment firstFragment, Fragment lastFragment) { @@ -33,13 +36,6 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc return Chars.wrap(sb); } - @Override - protected int localOffsetTransform(int outOffset, boolean inclusive) { - // todo this would be pretty slow when we're in the middle of some escapes - // we could check save the fragment last accessed to speed it up, and look forwards & backwards - return inputOffsetAt(outOffset, firstFragment, inclusive); - } - @Override public Chars getText() { return text; @@ -52,17 +48,40 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc } - static int inputOffsetAt(int outputOffset, @Nullable Fragment firstFragment, boolean inclusive) { - Fragment f = firstFragment; + @Override + protected int localOffsetTransform(int outOffset, boolean inclusive) { + // todo this would be pretty slow when we're in the middle of some escapes + // we could check save the fragment last accessed to speed it up, and look forwards & backwards + return inputOffsetAt(outOffset, inclusive); + } + + private int inputOffsetAt(int outputOffset, boolean inclusive) { + Fragment f = this.lastAccessedFragment; if (f == null) { return outputOffset; } - while (f.next != null && f.outEnd() < outputOffset) { - f = f.next; + + if (!f.contains(outputOffset)) { + // Slow path, we must search for the fragment + // This optimisation is important, otherwise we have + // to search for very long times in some files + + if (f.outEnd() < outputOffset) { // search forward + while (f.next != null && f.outEnd() < outputOffset) { + f = f.next; + } + } else { // search backwards + while (f.prev != null && outputOffset <= f.outStart()) { + f = f.prev; + } + } + lastAccessedFragment = f; } + if (!inclusive && f.outEnd() == outputOffset) { if (f.next != null) { f = f.next; + lastAccessedFragment = f; // fallthrough } else { return f.outToIn(outputOffset) + 1; From 68257d2244aaa0cdcb30a3b50c9d9fc171785bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Tue, 17 Nov 2020 00:06:09 +0100 Subject: [PATCH 053/136] Make sure we use either chars or string --- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 53 ++++++++++--------- .../ast/impl/javacc/JavaccTokenDocument.java | 2 +- .../pmd/lang/ast/impl/javacc/StringPool.java | 4 +- .../pmd/lang/java/ast/Comment.java | 4 +- .../pmd/lang/java/ast/JavaParser.java | 3 +- .../pmd/lang/java/ast/JavaTokenDocument.java | 2 +- .../java/rule/security/TypeResTestRule.java | 2 + 7 files changed, 38 insertions(+), 32 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index 39f6ea5ca5..7b384eefb2 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -5,6 +5,7 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import net.sourceforge.pmd.lang.ast.GenericToken; +import net.sourceforge.pmd.util.document.Chars; import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.TextRegion; @@ -75,6 +76,19 @@ public class JavaccToken implements GenericToken { public JavaccToken specialToken; + // common constructor, with a CharSequence parameter + JavaccToken(int kind, CharSequence image, int startInclusive, int endExclusive, JavaccTokenDocument document) { + assert document != null : "Null document"; + assert image instanceof String || image instanceof Chars : "Null image"; + assert TextRegion.isValidRegion(startInclusive, endExclusive, document.getTextDocument()); + + this.kind = kind; + this.image = image; + this.startOffset = startInclusive; + this.endOffset = endExclusive; + this.document = document; + } + /** * Builds a new token of the specified kind. * @@ -84,19 +98,15 @@ public class JavaccToken implements GenericToken { * @param endExclusive End of the token in the text file (before translating escapes) * @param document Document owning the token */ - public JavaccToken(int kind, - CharSequence image, - int startInclusive, - int endExclusive, - JavaccTokenDocument document) { - assert document != null : "Null document"; - assert TextRegion.isValidRegion(startInclusive, endExclusive, document.getTextDocument()); + public JavaccToken(int kind, Chars image, int startInclusive, int endExclusive, JavaccTokenDocument document) { + this(kind, (CharSequence) image, startInclusive, endExclusive, document); + } - this.kind = kind; - this.image = image; - this.startOffset = startInclusive; - this.endOffset = endExclusive; - this.document = document; + /** + * Constructor with a {@link String} image (see {@link #JavaccToken(int, Chars, int, int, JavaccTokenDocument) the other ctor}). + */ + public JavaccToken(int kind, String image, int startInclusive, int endExclusive, JavaccTokenDocument document) { + this(kind, (CharSequence) image, startInclusive, endExclusive, document); } /** @@ -122,7 +132,13 @@ public class JavaccToken implements GenericToken { } @Override - public CharSequence getImageCs() { + public Chars getImageCs() { + // wrap it: it's zero cost (images are either Chars or String) and Chars has a nice API + return Chars.wrap(image); + } + + /** Either {@link Chars} or {@link String}. */ + CharSequence getImageInternal() { return image; } @@ -177,17 +193,6 @@ public class JavaccToken implements GenericToken { ); } - public JavaccToken withImage(CharSequence image) { - return new JavaccToken( - this.kind, - image, - this.startOffset, - this.endOffset, - this.document - ); - } - - /** * Returns a new token with the given kind, and all other parameters diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 76be813888..92c7713a9c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -179,7 +179,7 @@ public final class JavaccTokenDocument extends TokenDocument { } String computeImage(JavaccToken t) { - CharSequence imageCs = t.getImageCs(); + CharSequence imageCs = t.getImageInternal(); if (imageCs instanceof String) { return (String) imageCs; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java index ace5c6a188..bd06cdfe6b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java @@ -20,7 +20,7 @@ import net.sourceforge.pmd.util.document.Chars; public final class StringPool { // This will be constant-folded by the JIT, use false in production - private static final boolean COLLECT_STATS = false; + private static final boolean COLLECT_STATS = true; private static final Stats STATS = new Stats(); private static final String[] SINGLE_CHARS; @@ -96,7 +96,7 @@ public final class StringPool { System.err.println("Unpooled string length: " + notPooledAlloc + " chars (" + toSize(notPooledAlloc) + ")"); } - private String toSize(long charLen) { + private static String toSize(long charLen) { // Assuming all pooled chars are latin-1, so 1B in a compressed string (byte[], not char[]) // Before Java 9, it's twice as much if (charLen > (1 << 20)) { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/Comment.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/Comment.java index a4bbde6182..2be2c3f4ad 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/Comment.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/Comment.java @@ -46,7 +46,7 @@ public abstract class Comment extends AbstractJjtreeNode { @Override @Deprecated public String getImage() { - return super.getImage(); + return getToken().getImage(); } public final JavaccToken getToken() { @@ -75,7 +75,7 @@ public abstract class Comment extends AbstractJjtreeNode { * @return List of lines of the comments */ private List multiLinesIn() { - String[] lines = NEWLINES_PATTERN.split(getText()); + String[] lines = NEWLINES_PATTERN.split(token.getImageCs()); List filteredLines = new ArrayList<>(lines.length); for (String rawLine : lines) { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java index dc07926b0a..28bec7b696 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaParser.java @@ -5,11 +5,10 @@ package net.sourceforge.pmd.lang.java.ast; import net.sourceforge.pmd.lang.ast.AstInfo; -import net.sourceforge.pmd.lang.ast.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.java.ast.internal.LanguageLevelChecker; /** diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 734da5a486..85872ed7c2 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -89,7 +89,7 @@ final class JavaTokenDocument extends JavaccTokenDocument.TokenDocumentBehavior final int realKind; - GTToken(int kind, int realKind, CharSequence image, int startOffset, int endOffset, JavaccTokenDocument doc) { + GTToken(int kind, int realKind, String image, int startOffset, int endOffset, JavaccTokenDocument doc) { super(kind, image, startOffset, endOffset, doc); this.realKind = realKind; } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java index 23740b5777..04f26af417 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java @@ -8,6 +8,7 @@ import org.apache.commons.lang3.StringUtils; import org.checkerframework.checker.nullness.qual.NonNull; import net.sourceforge.pmd.RuleContext; +import net.sourceforge.pmd.lang.ast.impl.javacc.StringPool; import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit; import net.sourceforge.pmd.lang.java.ast.JavaNode; import net.sourceforge.pmd.lang.java.ast.TypeNode; @@ -126,6 +127,7 @@ public class TypeResTestRule extends AbstractJavaRule { if (fid % 400 == 0) { synchronized (STATIC) { if (STATIC.fileId % 400 == 0) { + StringPool.printStats(); STATIC.print(); } } From f515e34d51e9746deeac4b7b9f7ce54f271540ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Tue, 17 Nov 2020 00:07:10 +0100 Subject: [PATCH 054/136] Remove string pool --- .../pmd/lang/ast/impl/javacc/JavaccToken.java | 2 +- .../ast/impl/javacc/JavaccTokenDocument.java | 9 -- .../pmd/lang/ast/impl/javacc/StringPool.java | 131 ------------------ .../java/rule/security/TypeResTestRule.java | 2 - 4 files changed, 1 insertion(+), 143 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java index 7b384eefb2..e70e8a8b6e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccToken.java @@ -144,7 +144,7 @@ public class JavaccToken implements GenericToken { @Override public String getImage() { - return document.computeImage(this); + return image.toString(); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 92c7713a9c..ecbaff0bbe 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -26,7 +26,6 @@ import net.sourceforge.pmd.util.document.TextDocument; public final class JavaccTokenDocument extends TokenDocument { private final TokenDocumentBehavior behavior; - private final StringPool stringPool = new StringPool(); private JavaccToken first; @@ -178,14 +177,6 @@ public final class JavaccTokenDocument extends TokenDocument { return first.next; } - String computeImage(JavaccToken t) { - CharSequence imageCs = t.getImageInternal(); - if (imageCs instanceof String) { - return (String) imageCs; - } - return stringPool.toString(imageCs, behavior.isImagePooled(t)); - } - /** * @see TokenDocumentBehavior#describeKind(int) */ diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java deleted file mode 100644 index bd06cdfe6b..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/StringPool.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.lang.ast.impl.javacc; - - -import java.util.LongSummaryStatistics; -import java.util.Map; -import java.util.WeakHashMap; - -import net.sourceforge.pmd.util.document.Chars; - -/** - * Simple pooling mechanism. Two {@link Chars} are equal if their - * charsequence is equal. Note that most {@link Chars} instances - * backing tokens hold a strong reference to the file's entire string - * representation, so a pool must not be reused between several files. - */ -public final class StringPool { - - // This will be constant-folded by the JIT, use false in production - private static final boolean COLLECT_STATS = true; - private static final Stats STATS = new Stats(); - - private static final String[] SINGLE_CHARS; - - static { - // all ascii characters - SINGLE_CHARS = new String[128]; - for (char i = 0; i < 128; i++) { - SINGLE_CHARS[i] = String.valueOf(i); - } - } - - - private final Map pool = new WeakHashMap<>(); - - String toString(CharSequence c, boolean doPool) { - if (c.length() == 1) { - char fst = c.charAt(0); - if (fst < 128) { - if (COLLECT_STATS) { - STATS.addCacheHit(1); - } - return SINGLE_CHARS[fst]; - } - } - - if (doPool && c instanceof Chars) { - if (COLLECT_STATS) { - return pool.compute( - (Chars) c, - (chars, s) -> { - if (s != null) { - STATS.addCacheHit(s.length()); - return s; - } else { - STATS.addCacheMiss(chars.length()); - return chars.toString(); - } - }); - } else { - return pool.computeIfAbsent((Chars) c, Chars::toString); - } - } - if (COLLECT_STATS) { - STATS.addPass(c); - } - return c.toString(); - } - - public static void printStats() { - if (COLLECT_STATS) { - STATS.print(); - } - } - - private static class Stats { - - long hits; - long miss; - final LongSummaryStatistics poolContents = new LongSummaryStatistics(); - long hitLen; - long totalTotal; - - long notPooledAlloc; - - public void print() { - System.err.println("String pool stats"); - System.err.println("================="); - System.err.println("Hits: " + hits + " (" + (hits * 100 / (total() + 1)) + "% of " + total() + ")"); - System.err.println("Hit length (net savings): " + hitLen + " (" + toSize(hitLen) + ")"); - System.err.println("Total pool size: " + poolContents.getCount() + " strings (" + toSize(poolContents.getSum()) + ")"); - System.err.println("Avg pooled string length: " + poolContents.getAverage() + " chars"); - System.err.println("Unpooled string length: " + notPooledAlloc + " chars (" + toSize(notPooledAlloc) + ")"); - } - - private static String toSize(long charLen) { - // Assuming all pooled chars are latin-1, so 1B in a compressed string (byte[], not char[]) - // Before Java 9, it's twice as much - if (charLen > (1 << 20)) { - return (charLen >> 20) + " MB"; - } else if (charLen > (1 << 10)) { - return (charLen >> 10) + " kB"; - } - return charLen + " B"; - } - - void addCacheMiss(int len) { - miss++; - poolContents.accept(len); - } - - void addCacheHit(int len) { - hits++; - hitLen += len; - } - - long total() { - return hits + miss; - } - - void addPass(CharSequence c) { - totalTotal++; - if (!(c instanceof String)) { - notPooledAlloc += c.length(); - } - } - } -} diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java index 04f26af417..23740b5777 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/security/TypeResTestRule.java @@ -8,7 +8,6 @@ import org.apache.commons.lang3.StringUtils; import org.checkerframework.checker.nullness.qual.NonNull; import net.sourceforge.pmd.RuleContext; -import net.sourceforge.pmd.lang.ast.impl.javacc.StringPool; import net.sourceforge.pmd.lang.java.ast.ASTCompilationUnit; import net.sourceforge.pmd.lang.java.ast.JavaNode; import net.sourceforge.pmd.lang.java.ast.TypeNode; @@ -127,7 +126,6 @@ public class TypeResTestRule extends AbstractJavaRule { if (fid % 400 == 0) { synchronized (STATIC) { if (STATIC.fileId % 400 == 0) { - StringPool.printStats(); STATIC.print(); } } From 870e13ce83656a62b2d854ed7bdc3780eedddf69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 18 Nov 2020 12:27:55 +0100 Subject: [PATCH 055/136] Cleanup duplicate code paths in escape translators --- .../ast/impl/javacc/JavaccTokenDocument.java | 16 +------ .../ast/impl/javacc/io/EscapeTranslator.java | 43 ++++++++++++------- .../impl/javacc/io/CharStreamImplTest.java | 7 ++- .../impl/javacc/io/JavaEscapeReaderTest.java | 4 +- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 8 +--- .../pmd/lang/java/ast/JavaTokenDocument.java | 7 +-- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 11 +++-- 7 files changed, 49 insertions(+), 47 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index ecbaff0bbe..8c9e3431dc 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import java.util.Collections; import java.util.List; -import java.util.function.Function; import org.checkerframework.checker.nullness.qual.NonNull; import org.checkerframework.checker.nullness.qual.Nullable; @@ -36,22 +35,11 @@ public final class JavaccTokenDocument extends TokenDocument { public static class TokenDocumentBehavior { - public static final TokenDocumentBehavior DEFAULT = new TokenDocumentBehavior(); + public static final TokenDocumentBehavior DEFAULT = new TokenDocumentBehavior(Collections.emptyList()); private final List tokenNames; - private final Function translator; - - private TokenDocumentBehavior() { - this(Collections.emptyList()); - } public TokenDocumentBehavior(List tokenNames) { - this(tokenNames, t -> t); - } - - public TokenDocumentBehavior(List tokenNames, - Function translator) { this.tokenNames = tokenNames; - this.translator = translator; } /** @@ -73,7 +61,7 @@ public final class JavaccTokenDocument extends TokenDocument { * @see EscapeTranslator */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { - return translator.apply(text); + return text; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java index c403724920..969d240853 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java @@ -6,8 +6,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; -import java.util.function.Function; - import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; @@ -22,7 +20,7 @@ import net.sourceforge.pmd.util.document.TextDocument; * perform any escape processing. Subclasses refine this behavior. */ @SuppressWarnings("PMD.AssignmentInOperand") -public abstract class EscapeTranslator implements AutoCloseable { +public abstract class EscapeTranslator { // Note that this can easily be turned into a java.io.Reader with // efficient block IO, optimized for the common case where there are // few or no escapes. This is part of the history of this file, but @@ -43,6 +41,13 @@ public abstract class EscapeTranslator implements AutoCloseable { private Chars curEscape; private int offInEscape; + /** + * Create a translator that will read from the given document. + * + * @param original Original document + * + * @throws NullPointerException If the parameter is null + */ public EscapeTranslator(TextDocument original) { AssertionUtil.requireParamNotNull("builder", original); this.input = original.getText(); @@ -52,10 +57,23 @@ public abstract class EscapeTranslator implements AutoCloseable { /** - * Translate all the input in the buffer. + * Translate all the input in the buffer. This consumes this object. + * + * @return The translated text document. If there is no escape, returns the original text + * + * @throws IllegalStateException If this method is called more than once on the same object + * @throws MalformedSourceException If there are invalid escapes in the source */ public TextDocument translateDocument() throws MalformedSourceException { ensureOpen(); + try { + return translateImpl(); + } finally { + close(); + } + } + + private TextDocument translateImpl() { if (this.bufpos == input.length()) { return escapes.build(); } @@ -113,15 +131,18 @@ public abstract class EscapeTranslator implements AutoCloseable { return startOffsetInclusive; } - @Override - public void close() { + /** + * Closing a translator does not close the underlying document, it just + * clears the intermediary state. + */ + private void close() { this.bufpos = -1; this.input = null; } /** Check to make sure that the stream has not been closed */ - protected void ensureOpen() { + protected final void ensureOpen() { if (input == null) { throw new IllegalStateException("Closed"); } @@ -142,12 +163,4 @@ public abstract class EscapeTranslator implements AutoCloseable { } - public static Function translatorFor(Function translatorMaker) { - return original -> { - try (EscapeTranslator translator = translatorMaker.apply(original)) { - return translator.translateDocument(); - } - }; - } - } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java index 725383344e..37208beccb 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java @@ -145,7 +145,12 @@ public class CharStreamImplTest { public CharStream javaCharStream(String abcd) { return CharStream.create(TextDocument.readOnlyString(abcd, dummyVersion), - new TokenDocumentBehavior(Collections.emptyList(), EscapeTranslator.translatorFor(JavaEscapeTranslator::new))); + new TokenDocumentBehavior(Collections.emptyList()) { + @Override + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + return new JavaEscapeTranslator(text).translateDocument(); + } + }); } @Test diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java index 8c9b803484..72ca3328b9 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java @@ -18,9 +18,7 @@ public class JavaEscapeReaderTest { public TextDocument readString(String input) { TextDocument intext = TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); - try (JavaEscapeTranslator translator = new JavaEscapeTranslator(intext)) { - return translator.translateDocument(); - } + return new JavaEscapeTranslator(intext).translateDocument(); } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 2dada00663..4c417481ac 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -66,13 +66,9 @@ public class CPPTokenizer extends JavaCCTokenizer { @Override protected TextDocument translate(TextDocument text) throws MalformedSourceException { if (skipBlocks) { - try (CppBlockSkipper translator = new CppBlockSkipper(text, skipBlocksStart, skipBlocksEnd)) { - text = translator.translateDocument(); - } - } - try (CppEscapeTranslator translator = new CppEscapeTranslator(text)) { - return translator.translateDocument(); + text = new CppBlockSkipper(text, skipBlocksStart, skipBlocksEnd).translateDocument(); } + return new CppEscapeTranslator(text).translateDocument(); } }; } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java index 85872ed7c2..c780ddce23 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocument.java @@ -17,7 +17,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.util.document.TextDocument; @@ -30,7 +29,7 @@ final class JavaTokenDocument extends JavaccTokenDocument.TokenDocumentBehavior static final JavaTokenDocument INSTANCE = new JavaTokenDocument(); private JavaTokenDocument() { - super(JavaTokenKinds.TOKEN_NAMES, EscapeTranslator.translatorFor(JavaEscapeTranslator::new)); + super(JavaTokenKinds.TOKEN_NAMES); } @@ -51,9 +50,7 @@ final class JavaTokenDocument extends JavaccTokenDocument.TokenDocumentBehavior @Override protected TextDocument translate(TextDocument text) throws MalformedSourceException { - try (JavaEscapeTranslator translator = new JavaEscapeTranslator(text)) { - return translator.translateDocument(); - } + return new JavaEscapeTranslator(text).translateDocument(); } diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 8ec128ac17..88d2a7f7c2 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -10,9 +10,10 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; +import net.sourceforge.pmd.util.document.TextDocument; /** * @author sergey.gorbaty @@ -26,8 +27,12 @@ public class VfTokenizer extends JavaCCTokenizer { @Override protected TokenDocumentBehavior tokenBehavior() { - return new JavaccTokenDocument.TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES, - EscapeTranslator.translatorFor(JavaEscapeTranslator::new)); + return new JavaccTokenDocument.TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES) { + @Override + protected TextDocument translate(TextDocument text) throws MalformedSourceException { + return new JavaEscapeTranslator(text).translateDocument(); + } + }; } } From 97e40a54b22c650ce4913ff0e59ad23ead73d56a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 18 Nov 2020 12:50:27 +0100 Subject: [PATCH 056/136] Improve FileLocation --- .../java/net/sourceforge/pmd/RuleContext.java | 2 +- .../pmd/cache/CachedRuleViolation.java | 2 +- .../ast/impl/javacc/io/EscapeTranslator.java | 27 ++++++------------- .../impl/javacc/io/JavaEscapeTranslator.java | 2 +- .../javacc/io/MalformedSourceException.java | 14 +++++----- .../pmd/util/document/FileLocation.java | 21 ++++++++++++--- .../util/document/FragmentedDocBuilder.java | 4 +-- .../pmd/util/document/TextDocument.java | 2 +- .../pmd/cache/FileAnalysisCacheTest.java | 2 +- .../token/internal/BaseTokenFilterTest.java | 2 +- .../pmd/lang/java/ast/JavadocElement.java | 2 +- .../pmd/lang/scala/ast/AbstractScalaNode.java | 10 +++---- .../pmd/test/lang/ast/DummyNode.java | 2 +- 13 files changed, 49 insertions(+), 43 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/RuleContext.java b/pmd-core/src/main/java/net/sourceforge/pmd/RuleContext.java index bf7e26c64b..16f2b73f5e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/RuleContext.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/RuleContext.java @@ -74,7 +74,7 @@ public final class RuleContext { FileLocation location = node.getReportLocation(); if (beginLine != -1 && endLine != -1) { - location = FileLocation.location(location.getFileName(), beginLine, 1, endLine, 1); + location = FileLocation.range(location.getFileName(), beginLine, 1, endLine, 1); } RuleViolation violation = fact.createViolation(rule, node, location, makeMessage(message, formatArgs)); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cache/CachedRuleViolation.java b/pmd-core/src/main/java/net/sourceforge/pmd/cache/CachedRuleViolation.java index f8f2da5430..032c7f5d74 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cache/CachedRuleViolation.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cache/CachedRuleViolation.java @@ -41,7 +41,7 @@ public final class CachedRuleViolation implements RuleViolation { final String className, final String methodName, final String variableName) { this.mapper = mapper; this.description = description; - this.location = FileLocation.location(fileName, beginLine, beginColumn, endLine, endColumn); + this.location = FileLocation.range(fileName, beginLine, beginColumn, endLine, endColumn); this.ruleClassName = ruleClassName; this.ruleName = ruleName; this.ruleTargetLanguage = ruleTargetLanguage; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java index 969d240853..b9dcb40fb8 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java @@ -7,8 +7,8 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; import static java.lang.Integer.min; import net.sourceforge.pmd.internal.util.AssertionUtil; -import net.sourceforge.pmd.util.StringUtil; import net.sourceforge.pmd.util.document.Chars; +import net.sourceforge.pmd.util.document.FileLocation; import net.sourceforge.pmd.util.document.FragmentedDocBuilder; import net.sourceforge.pmd.util.document.TextDocument; @@ -36,7 +36,7 @@ public abstract class EscapeTranslator { /** Position of the next char to read in the input. */ protected int bufpos; /** Keep track of adjustments to make to the offsets, caused by unicode escapes. */ - final FragmentedDocBuilder escapes; + final FragmentedDocBuilder builder; private Chars curEscape; private int offInEscape; @@ -52,7 +52,7 @@ public abstract class EscapeTranslator { AssertionUtil.requireParamNotNull("builder", original); this.input = original.getText(); this.bufpos = 0; - this.escapes = new FragmentedDocBuilder(original); + this.builder = new FragmentedDocBuilder(original); } @@ -75,7 +75,7 @@ public abstract class EscapeTranslator { private TextDocument translateImpl() { if (this.bufpos == input.length()) { - return escapes.build(); + return builder.build(); } final int len = input.length(); // remove Integer.MAX_VALUE @@ -108,7 +108,7 @@ public abstract class EscapeTranslator { } readChars += newlyReadChars; } - return escapes.build(); + return builder.build(); } /** @@ -124,7 +124,7 @@ public abstract class EscapeTranslator { protected int recordEscape(final int startOffsetInclusive, int endOffsetExclusive, Chars translation) { assert endOffsetExclusive > startOffsetInclusive && startOffsetInclusive >= 0; - this.escapes.recordDelta(startOffsetInclusive, endOffsetExclusive, translation); + this.builder.recordDelta(startOffsetInclusive, endOffsetExclusive, translation); this.bufpos = endOffsetExclusive; this.curEscape = translation; this.offInEscape = 0; @@ -148,19 +148,8 @@ public abstract class EscapeTranslator { } } - /** - * The parameter is an *input* offset. - */ - protected int getLine(int idxInInput) { - return StringUtil.lineNumberAt(input, idxInInput); + protected FileLocation locationAt(int indexInInput) { + return builder.toLocation(indexInInput); } - /** - * @see #getLine(int) - */ - protected int getColumn(int idxInInput) { - return StringUtil.columnNumberAt(input, idxInInput); - } - - } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java index 17c55043de..58e3378831 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java @@ -53,7 +53,7 @@ public final class JavaEscapeTranslator extends BackslashEscapeTranslator { return Chars.wrap(Character.toString(c)); } catch (NumberFormatException | IndexOutOfBoundsException e) { - throw new MalformedSourceException("Invalid unicode escape " + " at line", e, getLine(posOfFirstBackSlash), getColumn(posOfFirstBackSlash)); + throw new MalformedSourceException("Invalid unicode escape ", e, locationAt(posOfFirstBackSlash)); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java index b8210adb1c..e7e99b9e81 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java @@ -4,7 +4,10 @@ package net.sourceforge.pmd.lang.ast.impl.javacc.io; +import java.util.Objects; + import net.sourceforge.pmd.lang.ast.FileAnalysisException; +import net.sourceforge.pmd.util.document.FileLocation; /** * A {@link FileAnalysisException} thrown when the source format is invalid, @@ -12,18 +15,17 @@ import net.sourceforge.pmd.lang.ast.FileAnalysisException; */ public class MalformedSourceException extends FileAnalysisException { - private final int line; - private final int col; + private final FileLocation location; - public MalformedSourceException(String message, Throwable cause, int line, int col) { + public MalformedSourceException(String message, Throwable cause, FileLocation fileLocation) { super(message, cause); - this.line = line; - this.col = col; + this.location = Objects.requireNonNull(fileLocation); + setFileName(fileLocation.getFileName()); } @Override protected String positionToString() { - return super.positionToString() + " at line " + line + ", column " + col; + return super.positionToString() + " at " + location.startPosToString(); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FileLocation.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FileLocation.java index ed6be6558a..bef882cd03 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FileLocation.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FileLocation.java @@ -42,7 +42,7 @@ public final class FileLocation { private final int endColumn; private final String fileName; - /** @see #location(String, int, int, int, int) */ + /** @see #range(String, int, int, int, int) */ FileLocation(String fileName, int beginLine, int beginColumn, int endLine, int endColumn) { this.fileName = Objects.requireNonNull(fileName); this.beginLine = AssertionUtil.requireOver1("Begin line", beginLine); @@ -104,17 +104,32 @@ public final class FileLocation { } /** - * Creates a new location from the given parameters. + * Creates a new location for a range of text. * * @throws IllegalArgumentException If the file name is null * @throws IllegalArgumentException If any of the line/col parameters are strictly less than 1 * @throws IllegalArgumentException If the line and column are not correctly ordered * @throws IllegalArgumentException If the start offset or length are negative */ - public static FileLocation location(String fileName, int beginLine, int beginColumn, int endLine, int endColumn) { + public static FileLocation range(String fileName, int beginLine, int beginColumn, int endLine, int endColumn) { return new FileLocation(fileName, beginLine, beginColumn, endLine, endColumn); } + /** + * Returns a new location that starts and ends at the same position. + * + * @param fileName File name + * @param line Line number + * @param column Column number + * + * @return A new location + * + * @throws IllegalArgumentException See {@link #range(String, int, int, int, int)} + */ + public static FileLocation caret(String fileName, int line, int column) { + return new FileLocation(fileName, line, column, line, column); + } + @Override public String toString() { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java index 7c4397266d..30c83d71b1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/FragmentedDocBuilder.java @@ -22,8 +22,8 @@ public final class FragmentedDocBuilder { this.original = original; } - public Chars inputChars() { - return mainBuf; + public FileLocation toLocation(int indexInInput) { + return original.toLocation(TextRegion.caretAt(indexInInput)); } /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java index 0c4797d328..257638981c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/document/TextDocument.java @@ -179,7 +179,7 @@ public interface TextDocument extends Closeable { // todo doc default FileLocation createLocation(int bline, int bcol, int eline, int ecol) { - return FileLocation.location(getDisplayName(), bline, bcol, eline, ecol); + return FileLocation.range(getDisplayName(), bline, bcol, eline, ecol); } /** diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cache/FileAnalysisCacheTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cache/FileAnalysisCacheTest.java index 23adfd5212..0f1b45edb8 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cache/FileAnalysisCacheTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cache/FileAnalysisCacheTest.java @@ -114,7 +114,7 @@ public class FileAnalysisCacheTest { final RuleViolation rv = mock(RuleViolation.class); when(rv.getFilename()).thenReturn(sourceFile.getDisplayName()); - when(rv.getLocation()).thenReturn(FileLocation.location(sourceFile.getDisplayName(), 1, 2, 3, 4)); + when(rv.getLocation()).thenReturn(FileLocation.range(sourceFile.getDisplayName(), 1, 2, 3, 4)); final net.sourceforge.pmd.Rule rule = mock(net.sourceforge.pmd.Rule.class, Mockito.RETURNS_SMART_NULLS); when(rule.getLanguage()).thenReturn(mock(Language.class)); when(rv.getRule()).thenReturn(rule); diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java index 16bcdb6b01..275c8104f0 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/token/internal/BaseTokenFilterTest.java @@ -58,7 +58,7 @@ public class BaseTokenFilterTest { @Override public FileLocation getReportLocation() { - return FileLocation.location("n/a", 0, 0, 0, 0); + return FileLocation.range("n/a", 0, 0, 0, 0); } @Override diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavadocElement.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavadocElement.java index 7472a9aeb3..fbc50ea8c6 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavadocElement.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavadocElement.java @@ -16,7 +16,7 @@ public class JavadocElement extends Comment { public JavadocElement(JavaccToken t, int theBeginLine, int theEndLine, int theBeginColumn, int theEndColumn, JavadocTag theTag) { super(t); this.tag = theTag; - this.reportLoc = FileLocation.location("TODO", theBeginLine, theBeginColumn, theEndLine, theEndColumn); + this.reportLoc = FileLocation.range("TODO", theBeginLine, theBeginColumn, theEndLine, theEndColumn); } public JavadocTag tag() { diff --git a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/lang/scala/ast/AbstractScalaNode.java b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/lang/scala/ast/AbstractScalaNode.java index fe371267e4..8bb5e5b7c0 100644 --- a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/lang/scala/ast/AbstractScalaNode.java +++ b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/lang/scala/ast/AbstractScalaNode.java @@ -64,11 +64,11 @@ abstract class AbstractScalaNode extends AbstractNode { private FileLocation location; public void setCoords(int bline, int bcol, int eline, int ecol) { - this.location = FileLocation.location(":dummyFile:", bline, bcol, eline, ecol); + this.location = FileLocation.range(":dummyFile:", bline, bcol, eline, ecol); } @Override From fa31d54bd7b38bd1f54ebecc26d44523238de825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 8 Jul 2021 19:04:29 +0200 Subject: [PATCH 057/136] Fix tests --- .../net/sourceforge/pmd/lang/ast/Parser.java | 28 ++++++++++++++++--- .../pmd/processor/PmdRunnable.java | 1 + .../sourceforge/pmd/cpd/CppBlockSkipper.java | 2 +- .../pmd/cpd/CppEscapeTranslator.java | 3 +- .../sourceforge/pmd/cpd/JavaTokenizer.java | 3 +- .../pmd/lang/java/ast/ASTCompilationUnit.java | 2 +- .../pmd/lang/modelica/ast/ModelicaParser.java | 4 +-- .../sourceforge/pmd/cpd/PythonTokenizer.java | 3 -- 8 files changed, 31 insertions(+), 15 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Parser.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Parser.java index 4aeddec9ce..61d8479b62 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Parser.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Parser.java @@ -48,14 +48,17 @@ public interface Parser { private final TextDocument textDoc; private final SemanticErrorReporter reporter; - private final PropertySource propertySource; + private final ParserTaskProperties propertySource; public ParserTask(TextDocument textDoc, SemanticErrorReporter reporter) { + this(textDoc, reporter, new ParserTaskProperties()); + } + + private ParserTask(TextDocument textDoc, SemanticErrorReporter reporter, ParserTaskProperties source) { this.textDoc = Objects.requireNonNull(textDoc, "Text document was null"); this.reporter = Objects.requireNonNull(reporter, "reporter was null"); - this.propertySource = new ParserTaskProperties(); - propertySource.definePropertyDescriptor(COMMENT_MARKER); + this.propertySource = new ParserTaskProperties(source); } public static final PropertyDescriptor COMMENT_MARKER = @@ -114,12 +117,29 @@ public interface Parser { * Replace the text document with another. */ public ParserTask withTextDocument(TextDocument doc) { - return new ParserTask(doc, this.reporter); + return new ParserTask(doc, this.reporter, this.propertySource); } private static final class ParserTaskProperties extends AbstractPropertySource { + ParserTaskProperties() { + definePropertyDescriptor(COMMENT_MARKER); + } + + ParserTaskProperties(ParserTaskProperties toCopy) { + for (PropertyDescriptor prop : toCopy.getPropertyDescriptors()) { + definePropertyDescriptor(prop); + } + toCopy.getOverriddenPropertyDescriptors().forEach( + prop -> copyProperty(prop, toCopy, this) + ); + } + + static void copyProperty(PropertyDescriptor prop, PropertySource source, PropertySource target) { + target.setProperty(prop, source.getProperty(prop)); + } + @Override protected String getPropertySourceType() { return "ParserOptions"; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java b/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java index 4f2ee96385..1f0b25ac7b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java @@ -123,6 +123,7 @@ abstract class PmdRunnable implements Runnable { handler.declareParserTaskProperties(task.getProperties()); task.getProperties().setProperty(ParserTask.COMMENT_MARKER, configuration.getSuppressMarker()); + assert task.getCommentMarker().equals(configuration.getSuppressMarker()); Parser parser = handler.getParser(); diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java index 4770579f60..4dcb9273db 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java @@ -24,7 +24,7 @@ class CppBlockSkipper extends EscapeTranslator { return Pattern.compile("^(?i)" + Pattern.quote(marker), Pattern.MULTILINE); } - public CppBlockSkipper(TextDocument original, Pattern skipStartMarker, Pattern skipEndMarker) { + CppBlockSkipper(TextDocument original, Pattern skipStartMarker, Pattern skipEndMarker) { super(original); skipStart = skipStartMarker; skipEnd = skipEndMarker; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java index 5a4f0257aa..90bfd3576a 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java @@ -24,7 +24,8 @@ public class CppEscapeTranslator extends BackslashEscapeTranslator { if (input.charAt(off) == NEWLINE) { return recordEscape(backSlashOff, off + 1, Chars.EMPTY); } else if (input.charAt(off) == CARRIAGE_RETURN) { - if (input.charAt(++off) == NEWLINE) { + off++; + if (input.charAt(off) == NEWLINE) { return recordEscape(backSlashOff, off + 1, Chars.EMPTY); } } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index 39d1eb3699..5d769e12c8 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -13,12 +13,11 @@ import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.cpd.token.JavaCCTokenFilter; import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds; -import net.sourceforge.pmd.util.document.TextDocument; public class JavaTokenizer extends JavaCCTokenizer { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java index ae476de730..1b6ba66f57 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTCompilationUnit.java @@ -43,7 +43,7 @@ public final class ASTCompilationUnit extends AbstractJavaTypeNode implements Ja return astInfo; } - void setComments(List comments) { + void setComments(List comments) { this.comments = comments; } diff --git a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java index 560602b46a..27144242b5 100644 --- a/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java +++ b/pmd-modelica/src/main/java/net/sourceforge/pmd/lang/modelica/ast/ModelicaParser.java @@ -4,10 +4,8 @@ package net.sourceforge.pmd.lang.modelica.ast; -import org.checkerframework.checker.nullness.qual.Nullable; - -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.ParseException; +import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.ast.impl.javacc.JjtreeParserAdapter; diff --git a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java index 3ebcd5a7e1..c80d572f67 100644 --- a/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java +++ b/pmd-python/src/main/java/net/sourceforge/pmd/cpd/PythonTokenizer.java @@ -6,8 +6,6 @@ package net.sourceforge.pmd.cpd; import java.util.regex.Pattern; -import org.checkerframework.checker.nullness.qual.Nullable; - import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; @@ -15,7 +13,6 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.python.ast.PythonTokenKinds; -import net.sourceforge.pmd.util.document.TextDocument; /** * The Python tokenizer. From ab1e3f974e8c916bb84c141a31f669e87daf0869 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Thu, 8 Jul 2021 19:29:09 +0200 Subject: [PATCH 058/136] Fix plsql module --- .../pmd/lang/jsp/cpd/testdata/scriptletWithString.txt | 6 +++--- pmd-plsql/etc/grammar/PLSQL.jjt | 5 +++-- .../main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java | 2 +- .../net/sourceforge/pmd/lang/plsql/ast/ParsingExclusion.txt | 2 +- .../pmd/lang/plsql/ast/SqlPlusLexicalVariablesIssue195.txt | 2 +- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pmd-jsp/src/test/resources/net/sourceforge/pmd/lang/jsp/cpd/testdata/scriptletWithString.txt b/pmd-jsp/src/test/resources/net/sourceforge/pmd/lang/jsp/cpd/testdata/scriptletWithString.txt index a27643e225..cfd9afafa6 100644 --- a/pmd-jsp/src/test/resources/net/sourceforge/pmd/lang/jsp/cpd/testdata/scriptletWithString.txt +++ b/pmd-jsp/src/test/resources/net/sourceforge/pmd/lang/jsp/cpd/testdata/scriptletWithString.txt @@ -1,17 +1,17 @@ [Image] or [Truncated image[ Bcol Ecol L1 [<%--] 1 5 - [\nBSD-style license; for more info[ 5 1 + [\nBSD-style license; for more info[ 5 78 L3 [--%>] 1 5 L5 [<%] 1 3 - [\nString nodeContent = "<% %>";\n] 3 1 + [\nString nodeContent = "<% %>";\n] 3 31 L7 [%>] 1 3 L8 [<%] 1 3 - [\n] 1 3 EOF diff --git a/pmd-plsql/etc/grammar/PLSQL.jjt b/pmd-plsql/etc/grammar/PLSQL.jjt index 98a0b9389c..10e7ed7065 100644 --- a/pmd-plsql/etc/grammar/PLSQL.jjt +++ b/pmd-plsql/etc/grammar/PLSQL.jjt @@ -160,6 +160,7 @@ import net.sourceforge.pmd.lang.ast.Node; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.TokenMgrError; import net.sourceforge.pmd.lang.plsql.ast.internal.ParsingExclusion; +import net.sourceforge.pmd.util.document.Chars; public class PLSQLParserImpl { @@ -3472,7 +3473,7 @@ ASTPrimarySuffix PrimarySuffix() : ) ) | - ( arguments = Arguments() ) {sb.append(arguments) ; } + ( arguments = Arguments() ) {sb.append("Arguments") ; } // was broken before... ) { jjtThis.setImage(sb.toString()); return jjtThis; @@ -5234,6 +5235,7 @@ TOKEN : MORE : { <~["'"]> | <"'"> { + Chars image = input_stream.getTokenImageCs(); int quoteDelimiter = image.charAt(2); if (image.charAt(0) == 'n' || image.charAt(0) == 'N') { quoteDelimiter = image.charAt(3); @@ -5241,7 +5243,6 @@ TOKEN : int beforeQuote = image.charAt(image.length() - 2); if (quoteDelimiter == beforeQuote) { input_stream.backup(1); - image.setLength(image.length() - 1); SwitchTo(IN_STRING_LITERAL_TOKENIZE); } } diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index 32dbbf94c5..77abbf8794 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -8,8 +8,8 @@ import java.util.Properties; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.TokenManager; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.plsql.ast.PLSQLTokenKinds; public class PLSQLTokenizer extends JavaCCTokenizer { diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/ParsingExclusion.txt b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/ParsingExclusion.txt index 2bf8e5d5b6..396f9304a5 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/ParsingExclusion.txt +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/ParsingExclusion.txt @@ -1,4 +1,4 @@ -+- Input[@CanonicalImage = null, @ExcludedLinesCount = "6", @ExcludedRangesCount = "2", @Sourcecode = "begin\n do_something();\n -- pmd-exclude-begin: PMD does not like dbms_lob.trim (clash with TrimExpression)\n dbms_lob.trim(the_blob, 1000);\n -- pmd-exclude-end\n do_something_else(x);\nend;\n/\n\nselect dummy from dual a\nwhere 1=1\n -- pmd-exclude-begin: PMD does not like scalar subqueries in WHERE conditions\n and \'J\' = (select max(\'J\') from dual b)\n -- pmd-exclude-end\n;\n"] ++- Input[@CanonicalImage = null, @ExcludedLinesCount = "6", @ExcludedRangesCount = "2"] +- Global[@CanonicalImage = null] | +- Block[@CanonicalImage = null] | +- Statement[@CanonicalImage = null] diff --git a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SqlPlusLexicalVariablesIssue195.txt b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SqlPlusLexicalVariablesIssue195.txt index f655a96709..efec9a5183 100644 --- a/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SqlPlusLexicalVariablesIssue195.txt +++ b/pmd-plsql/src/test/resources/net/sourceforge/pmd/lang/plsql/ast/SqlPlusLexicalVariablesIssue195.txt @@ -1,3 +1,3 @@ -+- Input[@CanonicalImage = null, @ExcludedLinesCount = "0", @ExcludedRangesCount = "0", @Sourcecode = "\n-- see https://github.com/pmd/pmd/issues/195\n-- both define and spool are SQL*Plus commands, and they should not be ended with a semi-colon.\n\ndefine patch_name = acme_module\nspool &patch_name..log\n"] ++- Input[@CanonicalImage = null, @ExcludedLinesCount = "0", @ExcludedRangesCount = "0"] +- SqlPlusCommand[@CanonicalImage = "DEFINE PATCH_NAME = ACME_MODULE", @Image = "define patch_name = acme_module "] +- SqlPlusCommand[@CanonicalImage = "SPOOL &PATCH_NAME. . LOG", @Image = "spool &patch_name. . log "] From 2c0a641b2cbd82c34e950ceb4b435d55928bf53c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Mon, 7 Mar 2022 20:16:22 +0100 Subject: [PATCH 059/136] Cleanups --- .../pmd/cpd/internal/JavaCCTokenizer.java | 2 +- .../{io => }/BackslashEscapeTranslator.java | 2 +- .../pmd/lang/ast/impl/javacc/CharStream.java | 1 - .../javacc/{io => }/EscapeTranslator.java | 2 +- .../javacc/{io => }/JavaEscapeTranslator.java | 2 +- .../ast/impl/javacc/JavaccTokenDocument.java | 2 - .../{io => }/MalformedSourceException.java | 2 +- ...treamImplTest.java => CharStreamTest.java} | 113 ++++++------------ .../javacc/{io => }/JavaEscapeReaderTest.java | 17 +-- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 2 +- .../sourceforge/pmd/cpd/CppBlockSkipper.java | 4 +- .../pmd/cpd/CppEscapeTranslator.java | 2 +- .../java/ast/JavaTokenDocumentBehavior.java | 4 +- .../pmd/lang/java/ast/ParserCornersTest.java | 2 +- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 4 +- 15 files changed, 62 insertions(+), 99 deletions(-) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io => }/BackslashEscapeTranslator.java (97%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io => }/EscapeTranslator.java (98%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io => }/JavaEscapeTranslator.java (98%) rename pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io => }/MalformedSourceException.java (94%) rename pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io/CharStreamImplTest.java => CharStreamTest.java} (72%) rename pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/{io => }/JavaEscapeReaderTest.java (76%) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java index 2c6b840242..423008868c 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/internal/JavaCCTokenizer.java @@ -17,7 +17,7 @@ import net.sourceforge.pmd.lang.ast.FileAnalysisException; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.document.CpdCompat; import net.sourceforge.pmd.lang.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeTranslator.java similarity index 97% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeTranslator.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeTranslator.java index 8d73be143c..4b9c0b258f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/BackslashEscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/BackslashEscapeTranslator.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import static java.lang.Integer.min; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index a56f7bdb1d..b1a5c8f0d1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -8,7 +8,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import java.io.EOFException; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.FileLocation; import net.sourceforge.pmd.lang.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTranslator.java similarity index 98% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTranslator.java index 01b7f6c685..30431d995d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/EscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/EscapeTranslator.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import static java.lang.Integer.min; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java similarity index 98% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java index dae5e1968e..a51f5ff08a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.TextDocument; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 9f832d59cf..3538b6d4ff 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -12,8 +12,6 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.cpd.internal.JavaCCTokenizer; import net.sourceforge.pmd.lang.ast.impl.TokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; import net.sourceforge.pmd.lang.document.TextDocument; /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/MalformedSourceException.java similarity index 94% rename from pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java rename to pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/MalformedSourceException.java index 1708a442e1..bc3471861e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/MalformedSourceException.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/MalformedSourceException.java @@ -2,7 +2,7 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import java.util.Objects; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java similarity index 72% rename from pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java index fb371404fe..d88fb446f8 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/CharStreamImplTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java @@ -2,29 +2,24 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.fail; +import static org.junit.Assert.assertThrows; import java.io.EOFException; import java.io.IOException; import java.util.Collections; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.ExpectedException; import net.sourceforge.pmd.lang.LanguageRegistry; import net.sourceforge.pmd.lang.LanguageVersion; -import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.document.TextDocument; -public class CharStreamImplTest { +public class CharStreamTest { - @Rule - public ExpectedException expect = ExpectedException.none(); private LanguageVersion dummyVersion = LanguageRegistry.getDefaultLanguage().getDefaultVersion(); @Test @@ -32,31 +27,10 @@ public class CharStreamImplTest { CharStream stream = simpleCharStream(""); - expect.expect(EOFException.class); + assertThrows(EOFException.class, stream::readChar); - try { - stream.readChar(); - } catch (Exception e) { - assertEquals(stream.getStartOffset(), 0); - assertEquals(stream.getEndOffset(), 0); - throw e; - } - } - - @Test - public void testReadEofChars() throws IOException { - - CharStream stream = simpleCharStream(""); - - expect.expect(EOFException.class); - - try { - stream.readChar(); - } catch (Exception e) { - assertEquals(0, stream.getStartOffset()); - assertEquals(0, stream.getEndOffset()); - throw e; - } + assertEquals(stream.getStartOffset(), 0); + assertEquals(stream.getEndOffset(), 0); } @Test @@ -65,12 +39,7 @@ public class CharStreamImplTest { CharStream stream = simpleCharStream(""); for (int i = 0; i < 3; i++) { - try { - stream.readChar(); - fail(); - } catch (EOFException ignored) { - - } + assertThrows(EOFException.class, stream::readChar); } } @@ -85,8 +54,7 @@ public class CharStreamImplTest { assertEquals('c', stream.readChar()); assertEquals('d', stream.readChar()); - expect.expect(EOFException.class); - stream.readChar(); + assertThrows(EOFException.class, stream::readChar); } @Test @@ -106,8 +74,7 @@ public class CharStreamImplTest { assertEquals('d', stream.readChar()); - expect.expect(EOFException.class); - stream.readChar(); + assertThrows(EOFException.class, stream::readChar); } @Test @@ -135,10 +102,38 @@ public class CharStreamImplTest { assertEquals("\u00a0_", stream.getTokenImage()); - expect.expect(EOFException.class); - stream.readChar(); + assertThrows(EOFException.class, stream::readChar); } + @Test + public void testBacktrackTooMuch() throws IOException { + + CharStream stream = simpleCharStream("abcd"); + + assertEquals('a', stream.readChar()); + assertEquals('b', stream.readChar()); + assertEquals('c', stream.markTokenStart()); + assertEquals('d', stream.readChar()); + + stream.backup(2); // ok + + assertThrows(IllegalArgumentException.class, () -> stream.backup(1)); + } + + @Test + public void testBacktrackTooMuch2() throws IOException { + + CharStream stream = simpleCharStream("abcd"); + + assertEquals('a', stream.markTokenStart()); + assertEquals('b', stream.readChar()); + assertEquals('c', stream.readChar()); + assertEquals('d', stream.readChar()); + + assertThrows(IllegalArgumentException.class, () -> stream.backup(10)); + } + + public CharStream simpleCharStream(String abcd) { return CharStream.create(TextDocument.readOnlyString(abcd, dummyVersion), TokenDocumentBehavior.DEFAULT); } @@ -154,34 +149,4 @@ public class CharStreamImplTest { }); } - @Test - public void testBacktrackTooMuch() throws IOException { - - CharStream stream = simpleCharStream("abcd"); - - assertEquals('a', stream.readChar()); - assertEquals('b', stream.readChar()); - assertEquals('c', stream.markTokenStart()); - assertEquals('d', stream.readChar()); - - stream.backup(2); // ok - expect.expect(IllegalArgumentException.class); - stream.backup(1); - } - - @Test - public void testBacktrackTooMuch2() throws IOException { - - CharStream stream = simpleCharStream("abcd"); - - assertEquals('a', stream.markTokenStart()); - assertEquals('b', stream.readChar()); - assertEquals('c', stream.readChar()); - assertEquals('d', stream.readChar()); - - expect.expect(IllegalArgumentException.class); - stream.backup(10); - } - - } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java similarity index 76% rename from pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java rename to pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java index a59a7844e6..1d43033bd2 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/io/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java @@ -2,11 +2,12 @@ * BSD-style license; for more info see http://pmd.sourceforge.net/license.html */ -package net.sourceforge.pmd.lang.ast.impl.javacc.io; +package net.sourceforge.pmd.lang.ast.impl.javacc; + +import static org.junit.Assert.assertEquals; import java.io.IOException; -import org.junit.Assert; import org.junit.Test; import net.sourceforge.pmd.lang.LanguageRegistry; @@ -27,7 +28,7 @@ public class JavaEscapeReaderTest { String input = "abcdede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap(input), r.getText()); + assertEquals(Chars.wrap(input), r.getText()); } } @@ -36,7 +37,7 @@ public class JavaEscapeReaderTest { String input = "abc\\dede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap(input), r.getText()); + assertEquals(Chars.wrap(input), r.getText()); } } @@ -45,7 +46,7 @@ public class JavaEscapeReaderTest { String input = "abc\\\\\\dede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap(input), r.getText()); + assertEquals(Chars.wrap(input), r.getText()); } } @@ -54,7 +55,7 @@ public class JavaEscapeReaderTest { String input = "abc\\\\\\u00a0dede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap("abc\u00a0dede"), r.getText()); + assertEquals(Chars.wrap("abc\u00a0dede"), r.getText()); } } @@ -63,7 +64,7 @@ public class JavaEscapeReaderTest { String input = "abc\\\\\\u00a0d\\uu00a0ede"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap("abc\u00a0d\u00a0ede"), r.getText()); + assertEquals(Chars.wrap("abc\u00a0d\u00a0ede"), r.getText()); } } @@ -72,7 +73,7 @@ public class JavaEscapeReaderTest { String input = "abc\\\\\\u00a0dede\\u00a0"; try (TextDocument r = readString(input)) { - Assert.assertEquals(Chars.wrap("abc\u00a0dede\u00a0"), r.getText()); + assertEquals(Chars.wrap("abc\u00a0dede\u00a0"), r.getText()); } } } diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index 05926029a1..011871adde 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -15,7 +15,7 @@ import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.cpp.ast.CppTokenKinds; import net.sourceforge.pmd.lang.document.TextDocument; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java index b8810034af..35857ef375 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppBlockSkipper.java @@ -7,8 +7,8 @@ package net.sourceforge.pmd.cpd; import java.util.regex.Matcher; import java.util.regex.Pattern; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.EscapeTranslator; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.ast.impl.javacc.EscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.TextDocument; diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java index 32fbeddd1e..b89904ffaf 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CppEscapeTranslator.java @@ -4,7 +4,7 @@ package net.sourceforge.pmd.cpd; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.BackslashEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.BackslashEscapeTranslator; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.TextDocument; diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java index b157209464..c551808760 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java @@ -17,8 +17,8 @@ import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.document.TextDocument; /** diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java index 9d1d1818d7..d974ab3262 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/ParserCornersTest.java @@ -15,7 +15,7 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import net.sourceforge.pmd.lang.ast.ParseException; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.ast.test.BaseParsingHelper; import net.sourceforge.pmd.lang.java.BaseJavaTreeDumpTest; import net.sourceforge.pmd.lang.java.JavaParsingHelper; diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index a347c58d8f..198284c243 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -10,8 +10,8 @@ import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.JavaEscapeTranslator; -import net.sourceforge.pmd.lang.ast.impl.javacc.io.MalformedSourceException; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeTranslator; +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.document.TextDocument; import net.sourceforge.pmd.lang.vf.ast.VfTokenKinds; From 1da096b126fc4cc1a29e282bf84ce70aafa71d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 24 Apr 2022 11:50:33 +0200 Subject: [PATCH 060/136] Merge, wip --- .../net/sourceforge/pmd/lang/ast/Node.java | 6 +-- .../pmd/lang/document/BaseMappedDocument.java | 20 +++++++ .../lang/document/FragmentedTextDocument.java | 53 ++++++++++++++----- .../pmd/lang/document/TextDocument.java | 48 ++++++++++++++--- .../pmd/lang/document/TextPos2d.java | 5 ++ .../pmd/lang/document/TextRegion.java | 15 ++---- .../java/net/sourceforge/pmd/ReportTest.java | 5 +- .../pmd/RuleViolationComparatorTest.java | 2 +- .../pmd/renderers/AbstractRendererTest.java | 2 +- .../pmd/renderers/XMLRendererTest.java | 3 +- .../pmd/renderers/XSLTRendererTest.java | 2 +- 11 files changed, 117 insertions(+), 44 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java index 7d71a83c62..58860b614e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/Node.java @@ -130,12 +130,8 @@ public interface Node extends Reportable { * to create a region. */ default TextRegion getTextRegion() { - @SuppressWarnings("PMD.CloseResource") - TextDocument document = getAstInfo().getTextDocument(); FileLocation loc = getReportLocation(); - int startOffset = document.offsetAtLineColumn(loc.getStartPos()); - int endOffset = document.offsetAtLineColumn(loc.getStartPos()); - return TextRegion.fromBothOffsets(startOffset, endOffset); + return getAstInfo().getTextDocument().toRegion(loc.toRange2d()); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java index b25d6aaf33..cbea8633d8 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java @@ -59,6 +59,21 @@ abstract class BaseMappedDocument implements TextDocument { return base.createLineRange(startLineInclusive, endLineInclusive); } + @Override + public int offsetAtLineColumn(int line, int column) { + throw new UnsupportedOperationException(); + } + + @Override + public TextPos2d lineColumnAtOffset(int offset, boolean inclusive) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean isInRange(TextPos2d textPos2d) { + return false; + } + @Override public int inputOffset(int outOffset, boolean inclusive) { if (outOffset < 0 || outOffset > getLength()) { @@ -67,8 +82,13 @@ abstract class BaseMappedDocument implements TextDocument { return base.inputOffset(localOffsetTransform(outOffset, inclusive), inclusive); } + /** + * Output offset to input offset. + */ protected abstract int localOffsetTransform(int outOffset, boolean inclusive); + protected abstract int inverseLocalOffsetTransform(int inOffset, boolean inclusive); + @Override public void close() throws IOException { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java index 735c7aa2a3..99be79f6df 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.lang.document; +import java.util.function.ToIntFunction; + import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.LanguageVersion; @@ -48,44 +50,71 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc @Override protected int localOffsetTransform(int outOffset, boolean inclusive) { - return inputOffsetAt(outOffset, inclusive); + return offsetTransform(outOffset, inclusive, + Fragment::outToIn, + Fragment::outEnd, + Fragment::outStart + ); } - private int inputOffsetAt(int outputOffset, boolean inclusive) { + @Override + protected int inverseLocalOffsetTransform(int inOffset, boolean inclusive) { + return offsetTransform(inOffset, inclusive, + Fragment::inToOut, + Fragment::inEnd, + Fragment::inStart + ); + } + + interface OffsetMapper { + + int mapOffset(Fragment fragment, int offset); + } + + private int offsetTransform(int offset, + boolean inclusive, + OffsetMapper mapOffsetWhenContains, + ToIntFunction end, + ToIntFunction start) { // caching the last accessed fragment instead of doing // a linear search is critical for performance. Fragment f = this.lastAccessedFragment; if (f == null) { - return outputOffset; + return offset; } - if (!f.contains(outputOffset)) { + // Whether the fragment contains the offset we're looking for. + // Will be true most of the time. + boolean containsOffset = + start.applyAsInt(f) >= offset && offset < end.applyAsInt(f); + + if (!containsOffset) { // Slow path, we must search for the fragment // This optimisation is important, otherwise we have // to search for very long times in some files - if (f.outEnd() < outputOffset) { // search forward - while (f.next != null && f.outEnd() < outputOffset) { + if (end.applyAsInt(f) < offset) { // search forward + while (f.next != null && end.applyAsInt(f) < offset) { f = f.next; } } else { // search backwards - while (f.prev != null && outputOffset <= f.outStart()) { + while (f.prev != null && offset <= start.applyAsInt(f)) { f = f.prev; } } lastAccessedFragment = f; } - if (!inclusive && f.outEnd() == outputOffset) { + if (!inclusive && end.applyAsInt(f) == offset) { if (f.next != null) { f = f.next; lastAccessedFragment = f; // fallthrough } else { - return f.outToIn(outputOffset) + 1; + return mapOffsetWhenContains.mapOffset(f, offset) + 1; } } - return f.outToIn(outputOffset); + return mapOffsetWhenContains.mapOffset(f, offset); } @@ -150,8 +179,8 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc return inStart() + outOffset - outStart(); } - boolean contains(int outOffset) { - return outStart() <= outOffset && outEnd() > outOffset; + int inToOut(int inOffset) { + return inOffset - inStart() + outStart(); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java index e7fe94c960..10d5209fde 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java @@ -148,7 +148,7 @@ public interface TextDocument extends Closeable { int inputOffset(int outOffset, boolean inclusive); /** - * Translate a region given in the the coordinate system of this + * Translate a region given in the coordinate system of this * document, to the coordinate system of the original document. * This works as if creating a new region with both start and end * offsets translated through {@link #inputOffset(int, boolean)}. The @@ -160,6 +160,21 @@ public interface TextDocument extends Closeable { */ TextRegion inputRegion(TextRegion outputRegion); + /** + * Translate a 2D range given in the coordinate system of this + * document, to the coordinate system of the original document. + * This works as if creating a new region with both start and end + * offsets translated through {@link #inputOffset(int, boolean)}. The + * returned region may have a different length. + * + * @param outputRange Output region + * + * @return Input region + */ + default TextRange2d inputRange(TextRange2d outputRange) { + return toRange2d(inputRegion(toRegion(outputRange))); + } + /** * Returns a reader over the text of this document. @@ -176,16 +191,20 @@ public interface TextDocument extends Closeable { } /** - * Returns a text region that corresponds to the entire document. + * Returns a text region that corresponds to the entire document, + * in the coordinate system of this document. */ default TextRegion getEntireRegion() { return TextRegion.fromOffsetLength(0, getLength()); } /** - * Returns a 2D text range that corresponds to the entire document. + * Returns a 2D text range that corresponds to the entire document, + * in the coordinate system of this document. */ - TextRange2d getEntireRegion2d(); + default TextRange2d getEntireRegion2d() { + return toRange2d(getEntireRegion()); + } /** * Returns a region that spans the text of all the given lines. @@ -210,7 +229,9 @@ public interface TextDocument extends Closeable { * the line/column information for both start and end offset of * the region. * - * @return A new file position + * @param region A region, in the coordinate system of this document + * + * @return A new file position, with absolute coordinates * * @throws IndexOutOfBoundsException If the argument is not a valid region in this document */ @@ -231,7 +252,7 @@ public interface TextDocument extends Closeable { } TextRegion region = TextRegion.caretAt(startOffset); checkInRange(region, this.getLength()); - return FileLocation.location(getDisplayName(), range); + return FileLocation.range(getDisplayName(), range); } /** @@ -289,6 +310,8 @@ public interface TextDocument extends Closeable { /** * Returns the line and column at the given offset (inclusive). + * Both the input offset and the output range are in the coordinates + * of this document. * * @param offset A source offset (0-based), can range in {@code [0, length]}. * @param inclusive If the offset falls right after a line terminator, @@ -296,6 +319,8 @@ public interface TextDocument extends Closeable { * choose the position at the start of the next line. * Otherwise choose the offset at the end of the line. * + * @return A position, in the coordinate system of this document + * * @throws IndexOutOfBoundsException if the offset is out of bounds */ TextPos2d lineColumnAtOffset(int offset, boolean inclusive); @@ -315,6 +340,17 @@ public interface TextDocument extends Closeable { @Override void close() throws IOException; + /** + * Create a new text document for the given text file. The document's + * coordinate system is the same as the original text file. + * + * @param textFile A text file + * + * @return A new text document + * + * @throws IOException If the file cannot be read ({@link TextFile#readContents()}) + * @throws NullPointerException If the parameter is null + */ static TextDocument create(TextFile textFile) throws IOException { return new RootTextDocument(textFile); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java index bdf767af3b..4f561fb238 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java @@ -11,6 +11,11 @@ import org.checkerframework.checker.nullness.qual.NonNull; */ public final class TextPos2d implements Comparable { + /** + * The position at the start of the document (line 1, column 1). + */ + public static TextPos2d DOCUMENT_START = new TextPos2d(1, 1); + private final int line; private final int column; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextRegion.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextRegion.java index 5d14a965a2..a8fd7d6956 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextRegion.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextRegion.java @@ -122,8 +122,8 @@ public final class TextRegion implements Comparable { * @throws AssertionError If the parameter cannot produce a valid region */ public TextRegion growLeft(int delta) { - assert (delta + length) >= 0 : "Left delta " + delta + " would produce a negative length region" + parThis(); - assert (startOffset - delta) >= 0 : "Left delta " + delta + " would produce a region that starts before zero" + parThis(); + assert delta + length >= 0 : "Left delta " + delta + " would produce a negative length region" + parThis(); + assert startOffset - delta >= 0 : "Left delta " + delta + " would produce a region that starts before zero" + parThis(); return new TextRegion(startOffset - delta, delta + length); } @@ -135,7 +135,7 @@ public final class TextRegion implements Comparable { * @throws AssertionError If the delta is negative and less than the length of this region */ public TextRegion growRight(int delta) { - assert (delta + length) >= 0 : "Right delta " + delta + " would produce a negative length region" + parThis(); + assert delta + length >= 0 : "Right delta " + delta + " would produce a negative length region" + parThis(); return new TextRegion(startOffset, delta + length); } @@ -181,15 +181,6 @@ public final class TextRegion implements Comparable { return new TextRegion(startOffset, length); } - /** - * Builds a new region from offset and length. - * - * @throws AssertionError If either parameter is negative - */ - public static TextRegion caretAt(int offset) { - return fromOffsetLength(offset, 0); - } - /** * Builds a new region from start and end offset. * diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/ReportTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/ReportTest.java index 3bf7ea5247..f987287f8f 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/ReportTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/ReportTest.java @@ -75,10 +75,7 @@ public class ReportTest { } private static FileLocation getNode(int line, int column, String filename) { - return FileLocation.location( - filename, - TextRange2d.range2d(line, column, line, column) - ); + return FileLocation.range(filename, TextRange2d.range2d(line, column, line, column)); } public static String render(Renderer renderer, Consumer listenerEffects) throws IOException { diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/RuleViolationComparatorTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/RuleViolationComparatorTest.java index 84c00c96a3..cbfa83b685 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/RuleViolationComparatorTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/RuleViolationComparatorTest.java @@ -70,7 +70,7 @@ public class RuleViolationComparatorTest { private RuleViolation createJavaRuleViolation(Rule rule, String fileName, int beginLine, String description, int beginColumn, int endLine, int endColumn) { - FileLocation loc = FileLocation.location(fileName, TextRange2d.range2d(beginLine, beginColumn, endLine, endColumn)); + FileLocation loc = FileLocation.range(fileName, TextRange2d.range2d(beginLine, beginColumn, endLine, endColumn)); return new ParametricRuleViolation(rule, loc, description); } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/AbstractRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/AbstractRendererTest.java index 98b2b5a017..a61b2c0066 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/AbstractRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/AbstractRendererTest.java @@ -92,7 +92,7 @@ public abstract class AbstractRendererTest { protected FileLocation createLocation(int beginLine, int beginColumn, int endLine, int endColumn) { TextRange2d range2d = TextRange2d.range2d(beginLine, beginColumn, endLine, endColumn); - return FileLocation.location(getSourceCodeFilename(), range2d); + return FileLocation.range(getSourceCodeFilename(), range2d); } protected RuleViolation newRuleViolation(int beginLine, int beginColumn, int endLine, int endColumn, Rule rule) { diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java index 8cbe58a269..2ea3e05434 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XMLRendererTest.java @@ -91,8 +91,7 @@ public class XMLRendererTest extends AbstractRendererTest { } private RuleViolation createRuleViolation(String description) { - FileLocation loc = FileLocation.location(getSourceCodeFilename(), - TextRange2d.range2d(1, 1, 1, 1)); + FileLocation loc = FileLocation.range(getSourceCodeFilename(), TextRange2d.range2d(1, 1, 1, 1)); return new ParametricRuleViolation(new FooRule(), loc, description); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XSLTRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XSLTRendererTest.java index a376586054..48ac7089b9 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XSLTRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/renderers/XSLTRendererTest.java @@ -19,7 +19,7 @@ public class XSLTRendererTest { @Test public void testDefaultStylesheet() throws Exception { XSLTRenderer renderer = new XSLTRenderer(); - FileLocation loc = FileLocation.location("file", TextRange2d.range2d(1, 1, 1, 2)); + FileLocation loc = FileLocation.range("file", TextRange2d.range2d(1, 1, 1, 2)); RuleViolation rv = new ParametricRuleViolation(new FooRule(), loc, "violation message"); String result = ReportTest.render(renderer, it -> it.onRuleViolation(rv)); Assert.assertTrue(result.contains("violation message")); From d1d2f3011911497a11d6aec5733c6d79e9f51fee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 30 Apr 2022 15:50:40 +0200 Subject: [PATCH 061/136] Remove some things from TextDocument --- .../ast/impl/javacc/JavaccTokenDocument.java | 2 + .../pmd/lang/document/BaseMappedDocument.java | 14 +--- .../lang/document/FragmentedTextDocument.java | 45 ++-------- .../pmd/lang/document/TextDocument.java | 82 ------------------- .../pmd/lang/document/TextPos2d.java | 5 -- .../pmd/lang/document/TextDocumentTest.java | 2 +- pmd-java/etc/grammar/Java.jjt | 4 +- .../sourceforge/pmd/cpd/JavaTokenizer.java | 3 +- pmd-plsql/etc/grammar/PLSQL.jjt | 1 + .../pmd/cpd/ScalaTokenAdapter.java | 2 +- 10 files changed, 17 insertions(+), 143 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 3538b6d4ff..3dca87e381 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -57,6 +57,8 @@ public final class JavaccTokenDocument extends TokenDocument { * @param text Source doc * * @see EscapeTranslator + * + * TODO move that to LanguageVersionHandler once #3919 (Merge CPD and PMD language) is implemented */ protected TextDocument translate(TextDocument text) throws MalformedSourceException { return text; diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java index cbea8633d8..399f42f737 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java @@ -59,19 +59,9 @@ abstract class BaseMappedDocument implements TextDocument { return base.createLineRange(startLineInclusive, endLineInclusive); } - @Override - public int offsetAtLineColumn(int line, int column) { - throw new UnsupportedOperationException(); - } - @Override public TextPos2d lineColumnAtOffset(int offset, boolean inclusive) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean isInRange(TextPos2d textPos2d) { - return false; + return base.lineColumnAtOffset(localOffsetTransform(offset, inclusive)); } @Override @@ -87,8 +77,6 @@ abstract class BaseMappedDocument implements TextDocument { */ protected abstract int localOffsetTransform(int outOffset, boolean inclusive); - protected abstract int inverseLocalOffsetTransform(int inOffset, boolean inclusive); - @Override public void close() throws IOException { diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java index 99be79f6df..c3f372892a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java @@ -4,8 +4,6 @@ package net.sourceforge.pmd.lang.document; -import java.util.function.ToIntFunction; - import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.LanguageVersion; @@ -47,74 +45,47 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc return base.getLanguageVersion(); } - @Override protected int localOffsetTransform(int outOffset, boolean inclusive) { - return offsetTransform(outOffset, inclusive, - Fragment::outToIn, - Fragment::outEnd, - Fragment::outStart - ); - } - - @Override - protected int inverseLocalOffsetTransform(int inOffset, boolean inclusive) { - return offsetTransform(inOffset, inclusive, - Fragment::inToOut, - Fragment::inEnd, - Fragment::inStart - ); - } - - interface OffsetMapper { - - int mapOffset(Fragment fragment, int offset); - } - - private int offsetTransform(int offset, - boolean inclusive, - OffsetMapper mapOffsetWhenContains, - ToIntFunction end, - ToIntFunction start) { // caching the last accessed fragment instead of doing // a linear search is critical for performance. Fragment f = this.lastAccessedFragment; if (f == null) { - return offset; + return outOffset; } // Whether the fragment contains the offset we're looking for. // Will be true most of the time. boolean containsOffset = - start.applyAsInt(f) >= offset && offset < end.applyAsInt(f); + f.outStart() >= outOffset && outOffset < f.outEnd(); if (!containsOffset) { // Slow path, we must search for the fragment // This optimisation is important, otherwise we have // to search for very long times in some files - if (end.applyAsInt(f) < offset) { // search forward - while (f.next != null && end.applyAsInt(f) < offset) { + if (f.outEnd() < outOffset) { // search forward + while (f.next != null && f.outEnd() < outOffset) { f = f.next; } } else { // search backwards - while (f.prev != null && offset <= start.applyAsInt(f)) { + while (f.prev != null && outOffset <= f.outStart()) { f = f.prev; } } lastAccessedFragment = f; } - if (!inclusive && end.applyAsInt(f) == offset) { + if (!inclusive && f.outEnd() == outOffset) { if (f.next != null) { f = f.next; lastAccessedFragment = f; // fallthrough } else { - return mapOffsetWhenContains.mapOffset(f, offset) + 1; + return f.outToIn(outOffset) + 1; } } - return mapOffsetWhenContains.mapOffset(f, offset); + return f.outToIn(outOffset); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java index b2fe186cff..5003ed6a70 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java @@ -180,21 +180,6 @@ public interface TextDocument extends Closeable { */ TextRegion inputRegion(TextRegion outputRegion); - /** - * Translate a 2D range given in the coordinate system of this - * document, to the coordinate system of the original document. - * This works as if creating a new region with both start and end - * offsets translated through {@link #inputOffset(int, boolean)}. The - * returned region may have a different length. - * - * @param outputRange Output region - * - * @return Input region - */ - default TextRange2d inputRange(TextRange2d outputRange) { - return toRange2d(inputRegion(toRegion(outputRange))); - } - /** * Returns a reader over the text of this document. @@ -218,14 +203,6 @@ public interface TextDocument extends Closeable { return TextRegion.fromOffsetLength(0, getLength()); } - /** - * Returns a 2D text range that corresponds to the entire document, - * in the coordinate system of this document. - */ - default TextRange2d getEntireRegion2d() { - return toRange2d(getEntireRegion()); - } - /** * Returns a region that spans the text of all the given lines. * This is intended to provide a replacement for {@link SourceCode#getSlice(int, int)}. @@ -257,65 +234,6 @@ public interface TextDocument extends Closeable { */ FileLocation toLocation(TextRegion region); - /** - * Turn a text region into a {@link FileLocation}. The file name is - * the display name of this document. - * - * @return A new file position - * - * @throws IndexOutOfBoundsException If the argument is not a valid region in this document - */ - default FileLocation toLocation(TextRange2d range) { - int startOffset = offsetAtLineColumn(range.getStartPos()); - if (startOffset < 0) { - throw new IndexOutOfBoundsException("Region out of bounds: " + range.displayString()); - } - TextRegion region = TextRegion.caretAt(startOffset); - checkInRange(region, this.getLength()); - return FileLocation.range(getDisplayName(), range); - } - - /** - * Turn a text region to a {@link TextRange2d}. - */ - default TextRange2d toRange2d(TextRegion region) { - TextPos2d start = lineColumnAtOffset(region.getStartOffset(), true); - TextPos2d end = lineColumnAtOffset(region.getEndOffset(), false); - return TextRange2d.range2d(start, end); - } - - /** - * Turn a {@link TextRange2d} into a {@link TextRegion}. - */ - default TextRegion toRegion(TextRange2d region) { - return TextRegion.fromBothOffsets( - offsetAtLineColumn(region.getStartPos()), - offsetAtLineColumn(region.getEndPos()) - ); - } - - - /** - * Returns the offset at the given line and column number. - * - * @param line Line number (1-based) - * @param column Column number (1-based) - * - * @return an offset (0-based) - */ - int offsetAtLineColumn(int line, int column); - - /** - * Returns true if the position is valid in this document. - */ - boolean isInRange(TextPos2d textPos2d); - - /** - * Returns the offset at the line and number. - */ - default int offsetAtLineColumn(TextPos2d pos2d) { - return offsetAtLineColumn(pos2d.getLine(), pos2d.getColumn()); - } /** * Returns the line and column at the given offset (inclusive). diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java index 3a492f097a..b554cd3eda 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextPos2d.java @@ -11,11 +11,6 @@ import org.checkerframework.checker.nullness.qual.NonNull; */ public final class TextPos2d implements Comparable { - /** - * The position at the start of the document (line 1, column 1). - */ - public static TextPos2d DOCUMENT_START = new TextPos2d(1, 1); - private final int line; private final int column; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/TextDocumentTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/TextDocumentTest.java index af41bd22e0..0f7d2b9a91 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/TextDocumentTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/TextDocumentTest.java @@ -147,7 +147,7 @@ public class TextDocumentTest { } private void assertPos2dEqualsAt(TextDocument doc, int offset, String c, TextPos2d pos, boolean inclusive) { - Chars slicedChar = doc.sliceText(TextRegion.fromOffsetLength(offset, 1)); + Chars slicedChar = doc.sliceTranslatedText(TextRegion.fromOffsetLength(offset, 1)); assertEquals(c, slicedChar.toString()); assertEquals(pos, doc.lineColumnAtOffset(offset, inclusive)); } diff --git a/pmd-java/etc/grammar/Java.jjt b/pmd-java/etc/grammar/Java.jjt index 0ec6e94d12..dc698ede22 100644 --- a/pmd-java/etc/grammar/Java.jjt +++ b/pmd-java/etc/grammar/Java.jjt @@ -2618,14 +2618,14 @@ void AssertStatement() : void RUNSIGNEDSHIFT() #void: {} { - LOOKAHEAD({ JavaTokenDocument.getRealKind(getToken(1)) == RUNSIGNEDSHIFT}) + LOOKAHEAD({ JavaTokenDocumentBehavior.getRealKind(getToken(1)) == RUNSIGNEDSHIFT}) ">" ">" ">" } void RSIGNEDSHIFT() #void: {} { - LOOKAHEAD({ JavaTokenDocument.getRealKind(getToken(1)) == RSIGNEDSHIFT}) + LOOKAHEAD({ JavaTokenDocumentBehavior.getRealKind(getToken(1)) == RSIGNEDSHIFT}) ">" ">" } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java index 172be221fc..525d1731b2 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/cpd/JavaTokenizer.java @@ -15,7 +15,6 @@ import net.sourceforge.pmd.cpd.token.TokenFilter; import net.sourceforge.pmd.lang.TokenManager; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; -import net.sourceforge.pmd.lang.document.TextDocument; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; import net.sourceforge.pmd.lang.java.ast.InternalApiBridge; import net.sourceforge.pmd.lang.java.ast.JavaTokenKinds; @@ -44,7 +43,7 @@ public class JavaTokenizer extends JavaCCTokenizer { } @Override - protected JavaccTokenDocument.TokenDocumentBehavior tokenBehavior() throws IOException { + protected JavaccTokenDocument.TokenDocumentBehavior tokenBehavior() { return InternalApiBridge.javaTokenDoc(); } diff --git a/pmd-plsql/etc/grammar/PLSQL.jjt b/pmd-plsql/etc/grammar/PLSQL.jjt index 4688c275fc..41fbefea0c 100644 --- a/pmd-plsql/etc/grammar/PLSQL.jjt +++ b/pmd-plsql/etc/grammar/PLSQL.jjt @@ -160,6 +160,7 @@ package net.sourceforge.pmd.lang.plsql.ast; import java.util.List; import java.util.ArrayList; import net.sourceforge.pmd.lang.ast.Node; +import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; import net.sourceforge.pmd.lang.plsql.ast.internal.ParsingExclusion; import net.sourceforge.pmd.lang.ast.TokenMgrError; diff --git a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java index 06853bbe71..3609cd8efb 100644 --- a/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java +++ b/pmd-scala-modules/pmd-scala-common/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenAdapter.java @@ -44,7 +44,7 @@ public class ScalaTokenAdapter implements GenericToken { @Override public Chars getImageCs() { - return textDocument.sliceText(getRegion()); + return textDocument.sliceTranslatedText(getRegion()); } @Override From 158f0a6ee3731144d29cb976a34dc099a9c2144c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 30 Apr 2022 16:55:34 +0200 Subject: [PATCH 062/136] Fix bug with string literals --- .../pmd/lang/ast/TextAvailableNode.java | 41 ++++++++++----- .../ast/impl/javacc/AbstractJjtreeNode.java | 6 --- .../pmd/lang/ast/impl/javacc/JjtreeNode.java | 4 -- .../pmd/lang/document/BaseMappedDocument.java | 52 ++++++++++++++----- .../sourceforge/pmd/lang/document/Chars.java | 10 ++++ .../lang/document/FragmentedTextDocument.java | 2 +- .../pmd/lang/document/RootTextDocument.java | 10 ---- .../pmd/lang/document/TextDocument.java | 39 -------------- .../pmd/lang/document/CharsTest.java | 8 +++ .../pmd/lang/java/ast/ASTStringLiteral.java | 10 ++-- .../pmd/lang/java/ast/ASTLiteralTest.kt | 19 ++++++- 11 files changed, 110 insertions(+), 91 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java index 4d9841cb10..8f400cbfb6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/TextAvailableNode.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.lang.ast; +import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.TextDocument; import net.sourceforge.pmd.lang.document.TextRegion; import net.sourceforge.pmd.lang.rule.xpath.NoAttribute; @@ -18,27 +19,39 @@ public interface TextAvailableNode extends Node { /** - * Returns the exact region of text delimiting - * the node in the underlying text document. Note - * that {@link #getReportLocation()} does not need - * to match this region. {@link #getReportLocation()} - * can be scoped down to a specific token, eg the - * class identifier. This region uses the translated - * coordinate system, ie the coordinate system - * of {@link #getTextDocument()}. + * Returns the exact region of text delimiting the node in the underlying + * text document. Note that {@link #getReportLocation()} does not need + * to match this region. {@link #getReportLocation()} can be scoped down + * to a specific token, eg the class identifier. This region uses + * the translated coordinate system, ie the coordinate system of + * {@link #getTextDocument()}. */ @Override TextRegion getTextRegion(); /** - * Returns the original source code underlying this node. In particular, - * for a {@link RootNode}, returns the whole text of the file. Note the - * difference between this method and {@code getTextDocument().getText().slice(getTextRegion())}. - * The latter is {@link TextDocument#sliceTranslatedText(TextRegion)}, - * the former (this method) is {@link TextDocument#sliceOriginalText(TextRegion)}. + * Returns the original source code underlying this node, before + * any escapes have been translated. In particular, for a {@link RootNode}, + * returns the whole text of the file. + * + * @see TextDocument#sliceOriginalText(TextRegion) */ @NoAttribute - CharSequence getText(); + default Chars getOriginalText() { + return getTextDocument().sliceOriginalText(getTextRegion()); + } + + /** + * Returns the source code underlying this node, after any escapes + * have been translated. In particular, for a {@link RootNode}, returns + * the whole text of the file. + * + * @see TextDocument#sliceTranslatedText(TextRegion) + */ + @NoAttribute + default Chars getText() { + return getTextDocument().sliceTranslatedText(getTextRegion()); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java index 4097225d96..bd11611c28 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/AbstractJjtreeNode.java @@ -7,7 +7,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import net.sourceforge.pmd.annotation.Experimental; import net.sourceforge.pmd.lang.ast.Node; import net.sourceforge.pmd.lang.ast.impl.AbstractNode; -import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.FileLocation; import net.sourceforge.pmd.lang.document.TextRegion; import net.sourceforge.pmd.util.StringUtil; @@ -48,11 +47,6 @@ public abstract class AbstractJjtreeNode, N e this.image = image; } - @Override - public final Chars getText() { - return getTextDocument().sliceOriginalText(getTextRegion()); - } - @Override public final TextRegion getTextRegion() { return TextRegion.fromBothOffsets(getFirstToken().getStartOffset(), diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java index f7175cd2b3..0af5cfe313 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JjtreeNode.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.lang.ast.impl.javacc; import net.sourceforge.pmd.lang.ast.TextAvailableNode; import net.sourceforge.pmd.lang.ast.impl.GenericNode; -import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.reporting.Reportable; /** @@ -18,9 +17,6 @@ import net.sourceforge.pmd.reporting.Reportable; */ public interface JjtreeNode> extends GenericNode, TextAvailableNode, Reportable { - @Override - Chars getText(); - // todo token accessors should most likely be protected in PMD 7. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java index 399f42f737..d6e8fc9e8e 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/BaseMappedDocument.java @@ -45,14 +45,6 @@ abstract class BaseMappedDocument implements TextDocument { return base.toLocation(inputRegion(region)); } - @Override - public @NonNull TextRegion inputRegion(TextRegion outputRegion) { - // note that inputOffset already recurses up to the original document, - // so that we don't have to call base.inputRegion on the produced region - return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true), - inputOffset(outputRegion.getEndOffset(), false)); - } - @Override public TextRegion createLineRange(int startLineInclusive, int endLineInclusive) { // see the doc, lines do not need to be translated @@ -61,15 +53,51 @@ abstract class BaseMappedDocument implements TextDocument { @Override public TextPos2d lineColumnAtOffset(int offset, boolean inclusive) { - return base.lineColumnAtOffset(localOffsetTransform(offset, inclusive)); + return base.lineColumnAtOffset(inputOffset(offset, inclusive)); } - @Override - public int inputOffset(int outOffset, boolean inclusive) { + /** + * Translate a region given in the coordinate system of this + * document, to the coordinate system of the base document. + * This works as if creating a new region with both start and end + * offsets translated through {@link #inputOffset(int, boolean)}. The + * returned region may have a different length. + * + * @param outputRegion Output region + * + * @return Input region + */ + protected @NonNull TextRegion inputRegion(TextRegion outputRegion) { + return TextRegion.fromBothOffsets(inputOffset(outputRegion.getStartOffset(), true), + inputOffset(outputRegion.getEndOffset(), false)); + } + + /** + * Returns the input offset for the given output offset. This maps + * back an offset in the coordinate system of this document, to the + * coordinate system of the base document. This includes the + * length of any unicode escapes. + * + * 
+     * input:      "a\u00a0b"   (original document)
+     * translated: "a b"        (this document)
+     *
+     * translateOffset(0) = 0
+     * translateOffset(1) = 1
+     * translateOffset(2) = 7 // includes the length of the escape
+     *
+ * + * @param outOffset Output offset + * @param inclusive Whether the offset is to be interpreted as the index of a character (true), + * or the position after a character (false) + * + * @return Input offset + */ + protected final int inputOffset(int outOffset, boolean inclusive) { if (outOffset < 0 || outOffset > getLength()) { throw new IndexOutOfBoundsException(); } - return base.inputOffset(localOffsetTransform(outOffset, inclusive), inclusive); + return localOffsetTransform(outOffset, inclusive); } /** diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/Chars.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/Chars.java index 32834826c2..74f42f29c5 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/Chars.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/Chars.java @@ -588,6 +588,16 @@ public final class Chars implements CharSequence { return StreamSupport.stream(lines().spliterator(), false); } + /** + * Returns a new stringbuilder containing the whole contents of this + * char sequence. + */ + public StringBuilder toStringBuilder() { + StringBuilder sb = new StringBuilder(length()); + appendChars(sb); + return sb; + } + /** * Returns a new reader for the whole contents of this char sequence. diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java index c3f372892a..009fbbd30b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java @@ -57,7 +57,7 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc // Whether the fragment contains the offset we're looking for. // Will be true most of the time. boolean containsOffset = - f.outStart() >= outOffset && outOffset < f.outEnd(); + f.outStart() <= outOffset && outOffset < f.outEnd(); if (!containsOffset) { // Slow path, we must search for the fragment diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java index 56ccfcd223..8dc79889ec 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java @@ -119,16 +119,6 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { return content.getCheckSum(); } - @Override - public int inputOffset(int outOffset, boolean inclusive) { - return outOffset; - } - - @Override - public TextRegion inputRegion(TextRegion outputRegion) { - return outputRegion; - } - @Override public Chars sliceOriginalText(TextRegion region) { return getText().subSequence(region.getStartOffset(), region.getEndOffset()); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java index 5003ed6a70..153f43e360 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/TextDocument.java @@ -144,43 +144,6 @@ public interface TextDocument extends Closeable { long getCheckSum(); - /** - * Returns the input offset for the given output offset. This maps - * back an offset in the coordinate system of this document, to the - * coordinate system of the original document. This includes the - * length of any unicode escapes. - * - * 
-     * input:      "a\u00a0b"   (original document)
-     * translated: "a b"        (this document)
-     *
-     * translateOffset(0) = 0
-     * translateOffset(1) = 1
-     * translateOffset(2) = 7 // includes the length of the escape
-     *
- * - * @param outOffset Output offset - * @param inclusive Whether the offset is to be interpreted as the index of a character (true), - * or the position after a character (false) - * - * @return Input offset - */ - int inputOffset(int outOffset, boolean inclusive); - - /** - * Translate a region given in the coordinate system of this - * document, to the coordinate system of the original document. - * This works as if creating a new region with both start and end - * offsets translated through {@link #inputOffset(int, boolean)}. The - * returned region may have a different length. - * - * @param outputRegion Output region - * - * @return Input region - */ - TextRegion inputRegion(TextRegion outputRegion); - - /** * Returns a reader over the text of this document. */ @@ -261,8 +224,6 @@ public interface TextDocument extends Closeable { * * @return A position, in the coordinate system of the root document * - * @return A position, in the coordinate system of this document - * * @throws IndexOutOfBoundsException if the offset is out of bounds */ TextPos2d lineColumnAtOffset(int offset, boolean inclusive); diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/CharsTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/CharsTest.java index 9fd02c3620..9365670f3d 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/CharsTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/CharsTest.java @@ -60,6 +60,14 @@ public class CharsTest { assertEquals("b", sb.toString()); } + @Test + public void toStringBuilder() { + Chars bc = Chars.wrap("abcd").slice(1, 2); + assertEquals("bc", bc.toString()); + + assertEquals("bc", bc.toStringBuilder().toString()); + } + @Test public void write() throws IOException { StringWriter writer = new StringWriter(); diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java index be73eccec1..6ea1ceb380 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java @@ -9,7 +9,6 @@ import java.util.stream.Collectors; import org.apache.commons.lang3.StringEscapeUtils; import org.checkerframework.checker.nullness.qual.NonNull; -import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.util.CollectionUtil; @@ -31,6 +30,8 @@ public final class ASTStringLiteral extends AbstractLiteral implements ASTLitera } + // todo deprecate this + // it's ambiguous whether it returns getOriginalText or getTranslatedText @Override public String getImage() { return getText().toString(); @@ -72,12 +73,12 @@ public final class ASTStringLiteral extends AbstractLiteral implements ASTLitera } @Override - protected @Nullable Object buildConstValue() { + protected @NonNull Object buildConstValue() { if (isTextBlock()) { return determineTextBlockContent(getText()); } else { - CharSequence image = getText(); - CharSequence woDelims = image.subSequence(1, image.length() - 1); + Chars image = getText(); + Chars woDelims = image.subSequence(1, image.length() - 1); return StringEscapeUtils.UNESCAPE_JAVA.translate(woDelims); } } @@ -118,6 +119,7 @@ public final class ASTStringLiteral extends AbstractLiteral implements ASTLitera private static void interpretEscapeSequences(StringBuilder sb) { // interpret escape sequences "\" (line continuation), "n","t","b","r","f", "s", "\"", "\'", "\\" // we need to interpret everything in one pass, so regex replacement is inappropriate + // todo octal escapes: https://docs.oracle.com/javase/specs/jls/se17/html/jls-3.html#jls-EscapeSequence for (int i = 0; i < sb.length(); i++) { char c = sb.charAt(i); if (c == '\\' && i < sb.length() - 1) { diff --git a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt index c2829ae510..7aec741c03 100644 --- a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt +++ b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt @@ -4,6 +4,7 @@ package net.sourceforge.pmd.lang.java.ast +import io.kotest.matchers.shouldBe import net.sourceforge.pmd.lang.ast.test.NodeSpec import net.sourceforge.pmd.lang.ast.test.ValuedNodeSpec import net.sourceforge.pmd.lang.ast.test.shouldBe @@ -63,6 +64,7 @@ class ASTLiteralTest : ParserTestSpec({ } } } + suspend fun String.testTextBlock() { this.testTextBlock(EmptyAssertions) } @@ -166,7 +168,9 @@ $delim } "\"abc\\u1234abc\"" should parseAs { - stringLit("\"abc\\u1234abc\"") { + stringLit("\"abc\u1234abc\"") { + it.originalText.toString() shouldBe "\"abc\\u1234abc\"" + it.text.toString() shouldBe "\"abc\u1234abc\"" it::getConstValue shouldBe "abc\u1234abc" } } @@ -179,6 +183,19 @@ $delim } } + parserTest("String literal octal escapes") { + inContext(ExpressionParsingCtx) { + + "\"\\123\"" should parseAs { + stringLit("\"\\123\"") { + val char = "123".toInt(8).toChar() + it::getConstValue shouldBe char.toString() + } + } + + } + } + parserTest("Char literal") { From 87c224feaa504b75433a4a9ee5d704e755e30d02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 30 Apr 2022 20:59:55 +0200 Subject: [PATCH 063/136] Fix AvoidDuplicateLiteral message needs escaping --- .../java/net/sourceforge/pmd/util/StringUtil.java | 4 ++++ .../pmd/lang/java/ast/ASTStringLiteral.java | 12 ++++++++++++ .../rule/errorprone/AvoidDuplicateLiteralsRule.java | 3 +-- .../pmd/lang/java/ast/TextBlockEscapeTest.java | 8 ++------ .../sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt | 3 ++- 5 files changed, 21 insertions(+), 9 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java index 69215bb591..5e50749cb0 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/StringUtil.java @@ -13,6 +13,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import org.apache.commons.lang3.StringUtils; +import org.checkerframework.checker.nullness.qual.NonNull; import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.internal.util.AssertionUtil; @@ -543,6 +544,9 @@ public final class StringUtil { return str.replaceAll("'", "''"); } + public static @NonNull String inDoubleQuotes(String expected) { + return "\"" + expected + "\""; + } public enum CaseConvention { diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java index dcec3d611a..651eefe415 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/ASTStringLiteral.java @@ -10,6 +10,7 @@ import java.util.stream.Collectors; import org.checkerframework.checker.nullness.qual.NonNull; import net.sourceforge.pmd.lang.document.Chars; +import net.sourceforge.pmd.lang.rule.xpath.NoAttribute; import net.sourceforge.pmd.util.StringUtil; /** @@ -58,6 +59,17 @@ public final class ASTStringLiteral extends AbstractLiteral implements ASTLitera return getConstValue().length(); } + /** + * Returns a string where non-printable characters have been escaped + * using Java-like escape codes (eg \n, \t, \u005cu00a0). + */ + // ^^^^^^ + // this is a backslash, it's printed as \u00a0 + @NoAttribute + public @NonNull String toPrintableString() { + return StringUtil.inDoubleQuotes(StringUtil.escapeJava(getConstValue())); + } + @Override protected R acceptVisitor(JavaVisitor visitor, P data) { return visitor.visit(this, data); diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/AvoidDuplicateLiteralsRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/AvoidDuplicateLiteralsRule.java index b780b45a4c..3723278ed1 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/AvoidDuplicateLiteralsRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/errorprone/AvoidDuplicateLiteralsRule.java @@ -88,8 +88,7 @@ public class AvoidDuplicateLiteralsRule extends AbstractJavaRulechainRule { SortedSet occurrences = entry.getValue(); if (occurrences.size() >= threshold) { ASTStringLiteral first = occurrences.first(); - String rawImage = first.getImage(); - Object[] args = {rawImage, occurrences.size(), first.getBeginLine(), }; + Object[] args = { first.toPrintableString(), occurrences.size(), first.getBeginLine(), }; addViolation(data, first, args); } } diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/TextBlockEscapeTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/TextBlockEscapeTest.java index 1fb74b3099..1a40b50605 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/TextBlockEscapeTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/ast/TextBlockEscapeTest.java @@ -7,11 +7,11 @@ package net.sourceforge.pmd.lang.java.ast; import static net.sourceforge.pmd.lang.java.ast.ASTStringLiteral.determineTextBlockContent; import static org.junit.Assert.assertEquals; -import org.checkerframework.checker.nullness.qual.NonNull; import org.junit.Test; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.java.BaseParserTest; +import net.sourceforge.pmd.util.StringUtil; public class TextBlockEscapeTest extends BaseParserTest { @@ -42,15 +42,11 @@ public class TextBlockEscapeTest extends BaseParserTest { // note the argument order private void testStringEscape(String actual, String expected) { - actual = inDoubleQuotes(actual); + actual = StringUtil.inDoubleQuotes(actual); assertEquals(expected, ASTStringLiteral.determineStringContent(Chars.wrap(actual))); } - private static @NonNull String inDoubleQuotes(String expected) { - return "\"" + expected + "\""; - } - @Test public void testTextBlockContent() { assertEquals("single line text block", "winter", diff --git a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt index 909265b046..cdaae0c7d7 100644 --- a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt +++ b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/ASTLiteralTest.kt @@ -168,8 +168,9 @@ $delim } "\"abc\\u1234abc\"" should parseAs { - stringLit("\"abc\\u1234abc\"") { + stringLit("\"abc\u1234abc\"") { it::getConstValue shouldBe "abc\u1234abc" + it.originalText.toString() shouldBe "\"abc\\u1234abc\"" } } From 58ee669696b00ab3a9f55a9b82bf658c0943e870 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 30 Apr 2022 22:58:07 +0200 Subject: [PATCH 064/136] Add more tests for unicode escape --- .../ast/impl/javacc/JavaEscapeTranslator.java | 17 +++--- .../ast/impl/javacc/JavaccTokenDocument.java | 6 +- .../lang/ast/impl/javacc/CharStreamTest.java | 2 +- .../net/sourceforge/pmd/cpd/CPPTokenizer.java | 2 +- .../java/ast/JavaTokenDocumentBehavior.java | 8 +-- .../lang/java/ast/JavaUnicodeEscapesTest.kt | 58 +++++++++++++++++++ .../net/sourceforge/pmd/cpd/VfTokenizer.java | 2 +- 7 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/JavaUnicodeEscapesTest.kt diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java index a51f5ff08a..f230b15daa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeTranslator.java @@ -42,18 +42,21 @@ public final class JavaEscapeTranslator extends BackslashEscapeTranslator { } } - private Chars escapeValue(int posOfFirstBackSlash, int offOfTheU) throws MalformedSourceException { + private Chars escapeValue(int posOfFirstBackSlash, final int offOfTheU) throws MalformedSourceException { + int off = offOfTheU; try { char c = (char) - ( hexVal(input.charAt(++offOfTheU)) << 12 // SUPPRESS CHECKSTYLE paren pad - | hexVal(input.charAt(++offOfTheU)) << 8 - | hexVal(input.charAt(++offOfTheU)) << 4 - | hexVal(input.charAt(++offOfTheU)) - ); + ( hexVal(input.charAt(++off)) << 12 // SUPPRESS CHECKSTYLE paren pad + | hexVal(input.charAt(++off)) << 8 + | hexVal(input.charAt(++off)) << 4 + | hexVal(input.charAt(++off)) + ); return Chars.wrap(Character.toString(c)); } catch (NumberFormatException | IndexOutOfBoundsException e) { - throw new MalformedSourceException("Invalid unicode escape ", e, locationAt(posOfFirstBackSlash)); + // cut off u and 4 digits + String escape = input.substring(offOfTheU, Math.min(input.length(), offOfTheU + 5)); + throw new MalformedSourceException("Invalid unicode escape \\" + escape, e, locationAt(posOfFirstBackSlash)); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index 3dca87e381..d6f65185aa 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -60,15 +60,11 @@ public final class JavaccTokenDocument extends TokenDocument { * * TODO move that to LanguageVersionHandler once #3919 (Merge CPD and PMD language) is implemented */ - protected TextDocument translate(TextDocument text) throws MalformedSourceException { + public TextDocument translate(TextDocument text) throws MalformedSourceException { return text; } - protected boolean isImagePooled(JavaccToken t) { - return false; - } - /** * Returns a string that describes the token kind. * diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java index d88fb446f8..40e1740657 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java @@ -143,7 +143,7 @@ public class CharStreamTest { TextDocument.readOnlyString(abcd, dummyVersion), new TokenDocumentBehavior(Collections.emptyList()) { @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { + public TextDocument translate(TextDocument text) throws MalformedSourceException { return new JavaEscapeTranslator(text).translateDocument(); } }); diff --git a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java index c3a833369d..0728e9afbf 100644 --- a/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java +++ b/pmd-cpp/src/main/java/net/sourceforge/pmd/cpd/CPPTokenizer.java @@ -62,7 +62,7 @@ public class CPPTokenizer extends JavaCCTokenizer { return new TokenDocumentBehavior(CppTokenKinds.TOKEN_NAMES) { @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { + public TextDocument translate(TextDocument text) throws MalformedSourceException { if (skipBlocks) { text = new CppBlockSkipper(text, skipBlocksStart, skipBlocksEnd).translateDocument(); } diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java index c551808760..40b1ae9025 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java @@ -6,7 +6,6 @@ package net.sourceforge.pmd.lang.java.ast; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.FORMAL_COMMENT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.GT; -import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.IDENTIFIER; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.MULTI_LINE_COMMENT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.RSIGNEDSHIFT; import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.RUNSIGNEDSHIFT; @@ -49,16 +48,11 @@ final class JavaTokenDocumentBehavior extends JavaccTokenDocument.TokenDocumentB @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { + public TextDocument translate(TextDocument text) throws MalformedSourceException { return new JavaEscapeTranslator(text).translateDocument(); } - @Override - protected boolean isImagePooled(JavaccToken t) { - return t.kind == IDENTIFIER; - } - @Override public JavaccToken createToken(JavaccTokenDocument self, int kind, CharStream jcs, @Nullable String image) { switch (kind) { diff --git a/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/JavaUnicodeEscapesTest.kt b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/JavaUnicodeEscapesTest.kt new file mode 100644 index 0000000000..b84c3ba443 --- /dev/null +++ b/pmd-java/src/test/kotlin/net/sourceforge/pmd/lang/java/ast/JavaUnicodeEscapesTest.kt @@ -0,0 +1,58 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.java.ast + +import io.kotest.assertions.throwables.shouldThrow +import io.kotest.core.spec.style.FunSpec +import io.kotest.matchers.string.shouldContain +import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException +import net.sourceforge.pmd.lang.ast.test.IntelliMarker +import net.sourceforge.pmd.lang.ast.test.shouldBeA +import net.sourceforge.pmd.lang.document.TextDocument +import net.sourceforge.pmd.lang.java.JavaParsingHelper + +fun makeJavaTranslatedDocument( + code: String, +): TextDocument { + val base = TextDocument.readOnlyString( + code, + JavaParsingHelper.DEFAULT.defaultVersion + ) + return InternalApiBridge.javaTokenDoc().translate(base) +} + +class JavaUnicodeEscapesTest : IntelliMarker, FunSpec({ + + test("Test java invalid unicode escapes") { + + val comment = """\u002F\u0k2a\u002a\u002a\u002F""" + + val exception = shouldThrow { + makeJavaTranslatedDocument(comment) + } + + exception.message!!.shouldContain(Regex("line \\d+, column \\d+")) + exception.message!!.shouldContain("\\u0k2a") + + exception.cause!!.shouldBeA { + it.message!!.shouldContain("valid hexadecimal digit") + } + } + + test("Test incomplete unicode escape ") { + + val comment = """\u00""" + + val mse = shouldThrow { + makeJavaTranslatedDocument(comment) + } + mse.message!!.shouldContain(Regex("line \\d+, column \\d+")) + mse.message!!.shouldContain("\\u00") + mse.cause!!.shouldBeA() + } + +}) + + diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 7a7ddcda90..17926160b0 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -29,7 +29,7 @@ public class VfTokenizer extends JavaCCTokenizer { protected TokenDocumentBehavior tokenBehavior() { return new JavaccTokenDocument.TokenDocumentBehavior(VfTokenKinds.TOKEN_NAMES) { @Override - protected TextDocument translate(TextDocument text) throws MalformedSourceException { + public TextDocument translate(TextDocument text) throws MalformedSourceException { return new JavaEscapeTranslator(text).translateDocument(); } }; From 7b65280f36153f02cde458dc825efe612c6bc2d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sat, 30 Apr 2022 23:04:52 +0200 Subject: [PATCH 065/136] Doc --- .../pmd/lang/ast/impl/javacc/JavaccTokenDocument.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java index d6f65185aa..6b13d723d4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaccTokenDocument.java @@ -31,6 +31,9 @@ public final class JavaccTokenDocument extends TokenDocument { this.behavior = behavior; } + /** + * Overridable configuration of a token document. + */ public static class TokenDocumentBehavior { public static final TokenDocumentBehavior DEFAULT = new TokenDocumentBehavior(Collections.emptyList()); @@ -43,8 +46,9 @@ public final class JavaccTokenDocument extends TokenDocument { /** * Returns true if the lexer should accumulate the image of MORE * tokens into the StringBuilder jjimage. This is useless in our - * current implementations. The default returns false, which makes - * {@link CharStream#appendSuffix(StringBuilder, int)} a noop. + * current implementations, because the image of tokens can be cut + * out using text coordinates, so doesn't need to be put into a separate string. + * The default returns false, which makes {@link CharStream#appendSuffix(StringBuilder, int)} a noop. */ public boolean useMarkSuffix() { return false; From 1b5f7e8baedb07e0ea2067adb384cddf3e100dea Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 13 May 2022 19:56:47 +0200 Subject: [PATCH 066/136] [core] Remember forced language per file Fixes #3970 --- docs/pages/release_notes.md | 2 ++ .../sourceforge/pmd/SourceCodeProcessor.java | 2 +- .../pmd/lang/LanguageVersionDiscoverer.java | 28 ++++++++++++--- .../pmd/lang/document/FileCollector.java | 4 ++- .../net/sourceforge/pmd/PmdAnalysisTest.java | 35 +++++++++++++++++++ .../pmd/lang/Dummy2LanguageModule.java | 4 +++ .../resources/sample-source/dummy/foo.txt | 1 + 7 files changed, 70 insertions(+), 6 deletions(-) create mode 100644 pmd-core/src/test/resources/sample-source/dummy/foo.txt diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index c31a705241..1770a48672 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,6 +16,8 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* core + * [#3970](https://github.com/pmd/pmd/issues/3970): \[core] FileCollector.addFile ignores language parameter * javascript * [#3948](https://github.com/pmd/pmd/issues/3948): \[js] Invalid operator error for method property in object literal diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java b/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java index 7d1838c297..f52b0f2a2a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java @@ -222,7 +222,7 @@ public class SourceCodeProcessor { ctx.setLanguageVersion(forceLanguage); } else { // otherwise determine by file extension - LanguageVersion languageVersion = configuration.getLanguageVersionOfFile(ctx.getSourceCodeFilename()); + LanguageVersion languageVersion = configuration.getLanguageVersionOfFile(ctx.getSourceCodeFile().toString()); ctx.setLanguageVersion(languageVersion); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java index 4e22b03a4f..27c72898e1 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java @@ -5,6 +5,8 @@ package net.sourceforge.pmd.lang; import java.io.File; +import java.io.IOException; +import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -12,6 +14,7 @@ import java.util.Objects; import org.apache.commons.lang3.StringUtils; +import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.internal.util.AssertionUtil; /** @@ -24,6 +27,8 @@ public class LanguageVersionDiscoverer { private LanguageVersion forcedVersion; + private Map forcedVersionByFile = new HashMap<>(); + public LanguageVersionDiscoverer() { this(null); } @@ -97,10 +102,13 @@ public class LanguageVersionDiscoverer { * file. */ public LanguageVersion getDefaultLanguageVersionForFile(String fileName) { - List languages = getLanguagesForFile(fileName); - LanguageVersion languageVersion = null; - if (!languages.isEmpty()) { - languageVersion = getDefaultLanguageVersion(languages.get(0)); + LanguageVersion languageVersion = forcedVersionByFile.get(fileName); + + if (languageVersion == null) { + List languages = getLanguagesForFile(fileName); + if (!languages.isEmpty()) { + languageVersion = getDefaultLanguageVersion(languages.get(0)); + } } return languageVersion; } @@ -142,4 +150,16 @@ public class LanguageVersionDiscoverer { } + + @InternalApi + @Deprecated + public void recordLanguageVersionForFile(Path file, LanguageVersion languageVersion) { + String fileName; + try { + fileName = file.toRealPath().toString(); + } catch (IOException e) { + fileName = file.toAbsolutePath().toString(); + } + forcedVersionByFile.put(fileName, languageVersion); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java index 8db97175c3..51d84106c4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java @@ -155,8 +155,10 @@ public final class FileCollector implements AutoCloseable { reporter.error("Not a regular file {0}", file); return false; } - NioTextFile nioTextFile = new NioTextFile(file, charset, discoverer.getDefaultLanguageVersion(language), getDisplayName(file)); + LanguageVersion languageVersion = discoverer.getDefaultLanguageVersion(language); + NioTextFile nioTextFile = new NioTextFile(file, charset, languageVersion, getDisplayName(file)); addFileImpl(nioTextFile); + discoverer.recordLanguageVersionForFile(file, languageVersion); return true; } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java index 4ec5555cf9..797f4e0569 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java @@ -14,10 +14,17 @@ import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; +import org.junit.Assert; import org.junit.Test; import org.mockito.ArgumentMatchers; +import net.sourceforge.pmd.lang.Dummy2LanguageModule; +import net.sourceforge.pmd.lang.Language; +import net.sourceforge.pmd.lang.ast.Node; +import net.sourceforge.pmd.lang.rule.AbstractRule; import net.sourceforge.pmd.renderers.Renderer; /** @@ -72,4 +79,32 @@ public class PmdAnalysisTest { } } + @Test + public void testFileWithSpecificLanguage() { + final Language language = Dummy2LanguageModule.getInstance(); + PMDConfiguration config = new PMDConfiguration(); + config.setIgnoreIncrementalAnalysis(true); + RuleSet ruleset = RuleSet.forSingleRule(new TestRule()); + + try (PmdAnalysis pmd = PmdAnalysis.create(config)) { + pmd.addRuleSet(ruleset); + pmd.files().addFile(Paths.get("src", "test", "resources", "sample-source", "dummy", "foo.txt"), language); + Report report = pmd.performAnalysisAndCollectReport(); + Assert.assertEquals(0, report.getProcessingErrors().size()); + Assert.assertEquals(1, report.getViolations().size()); + } + } + + public static class TestRule extends AbstractRule { + public TestRule() { + setLanguage(Dummy2LanguageModule.getInstance()); + setMessage("dummy 2 test rule"); + } + + @Override + public void apply(List nodes, RuleContext ctx) { + ctx.addViolation(nodes.get(0)); + } + } + } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/Dummy2LanguageModule.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/Dummy2LanguageModule.java index e5a8de77cb..9d71d22010 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/Dummy2LanguageModule.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/Dummy2LanguageModule.java @@ -16,4 +16,8 @@ public class Dummy2LanguageModule extends BaseLanguageModule { super(NAME, null, TERSE_NAME, "dummy2"); addVersion("1.0", new DummyLanguageModule.Handler(), true); } + + public static Language getInstance() { + return LanguageRegistry.getLanguage(NAME); + } } diff --git a/pmd-core/src/test/resources/sample-source/dummy/foo.txt b/pmd-core/src/test/resources/sample-source/dummy/foo.txt new file mode 100644 index 0000000000..adaad87177 --- /dev/null +++ b/pmd-core/src/test/resources/sample-source/dummy/foo.txt @@ -0,0 +1 @@ +A dummy file with file extension txt. From 227e27c369e313a99f7fceeca2cf65fd48bdf47c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 17 Jul 2022 17:55:38 +0200 Subject: [PATCH 067/136] fix inclusive offsets in cpp test --- .../lang/document/FragmentedTextDocument.java | 13 ++++++------- .../cpd/testdata/continuation_intra_token.txt | 18 +++++++++--------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java index 009fbbd30b..d92404eeb4 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FragmentedTextDocument.java @@ -76,14 +76,13 @@ final class FragmentedTextDocument extends BaseMappedDocument implements TextDoc lastAccessedFragment = f; } - if (!inclusive && f.outEnd() == outOffset) { - if (f.next != null) { + if (inclusive && f.outEnd() == outOffset && f.next != null) { + // Inclusive means, the offset must correspond to a character in the source document. + // Here we have to skip forward to the fragment that contains the character, because + // it's not this one. + do { f = f.next; - lastAccessedFragment = f; - // fallthrough - } else { - return f.outToIn(outOffset) + 1; - } + } while (f.next != null && f.outLen() == 0); } return f.outToIn(outOffset); } diff --git a/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt b/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt index febb035697..df9e43234b 100644 --- a/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt +++ b/pmd-cpp/src/test/resources/net/sourceforge/pmd/lang/cpp/cpd/testdata/continuation_intra_token.txt @@ -1,14 +1,14 @@ [Image] or [Truncated image[ Bcol Ecol L1 [void] 1 2 -L5 - [main] 2 2 -L9 - [(] 2 2 +L6 + [main] 1 2 L10 - [)] 2 2 -L12 - [{] 2 2 -L14 - [}] 2 2 + [(] 1 2 +L11 + [)] 1 2 +L13 + [{] 1 2 +L15 + [}] 1 2 EOF From 34a668c0125594f5801eb4f9cbd0ffae52b9d015 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Sun, 17 Jul 2022 22:50:10 +0200 Subject: [PATCH 068/136] checkstyle --- .../pmd/lang/java/ast/JavaTokenDocumentBehavior.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java index 40b1ae9025..a6df00ae70 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/ast/JavaTokenDocumentBehavior.java @@ -14,9 +14,9 @@ import static net.sourceforge.pmd.lang.java.ast.JavaTokenKinds.SINGLE_LINE_COMME import org.checkerframework.checker.nullness.qual.Nullable; import net.sourceforge.pmd.lang.ast.impl.javacc.CharStream; +import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccToken; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument; -import net.sourceforge.pmd.lang.ast.impl.javacc.JavaEscapeTranslator; import net.sourceforge.pmd.lang.ast.impl.javacc.MalformedSourceException; import net.sourceforge.pmd.lang.document.TextDocument; From 84c5887f8867712f6c7cd4353e48a05e0cca98f8 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 19 Jul 2022 20:40:39 +0200 Subject: [PATCH 069/136] [doc] Fix missing additional rulesets Fixes #4051 --- docs/pages/release_notes.md | 1 + .../net/sourceforge/pmd/docs/GenerateRuleDocsCmd.java | 9 +++++---- .../net/sourceforge/pmd/docs/RuleSetResolverTest.java | 9 +++++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 56690a23a0..2084697dbf 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -43,6 +43,7 @@ Being based on a proper Antlr grammar, CPD can: ### Fixed Issues * core * [#4031](https://github.com/pmd/pmd/issues/4031): \[core] If report is written to stdout, stdout should not be closed + * [#4051](https://github.com/pmd/pmd/issues/4051): \[doc] Additional rulesets are not listed in documentation * java * [#4015](https://github.com/pmd/pmd/issues/4015): \[java] Support JDK 19 * java-bestpractices diff --git a/pmd-doc/src/main/java/net/sourceforge/pmd/docs/GenerateRuleDocsCmd.java b/pmd-doc/src/main/java/net/sourceforge/pmd/docs/GenerateRuleDocsCmd.java index d5b3002a6b..b434d4e8f5 100644 --- a/pmd-doc/src/main/java/net/sourceforge/pmd/docs/GenerateRuleDocsCmd.java +++ b/pmd-doc/src/main/java/net/sourceforge/pmd/docs/GenerateRuleDocsCmd.java @@ -47,16 +47,17 @@ public final class GenerateRuleDocsCmd { System.out.println("Generated docs in " + (System.currentTimeMillis() - start) + " ms"); } + static final Pattern ADDITIONAL_RULESET_PATTERN = Pattern.compile("^.+" + Pattern.quote(File.separator) + "pmd-\\w+" + + Pattern.quote(IOUtil.normalizePath(File.separator + Paths.get("src", "main", "resources", "rulesets").toString()) + File.separator) + + "\\w+" + Pattern.quote(File.separator) + "\\w+.xml$"); + public static List findAdditionalRulesets(Path basePath) { try { List additionalRulesets = new ArrayList<>(); - Pattern rulesetPattern = Pattern.compile("^.+" + Pattern.quote(File.separator) + "pmd-\\w+" - + Pattern.quote(IOUtil.normalizePath(File.separator + Paths.get("src", "main", "resources", "rulesets").toString() + File.separator)) - + "\\w+" + Pattern.quote(File.separator) + "\\w+.xml$"); Files.walkFileTree(basePath, new SimpleFileVisitor() { @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (rulesetPattern.matcher(file.toString()).matches()) { + if (ADDITIONAL_RULESET_PATTERN.matcher(file.toString()).matches()) { additionalRulesets.add(file.toString()); } diff --git a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java index 3d85da8daa..7894f8c97f 100644 --- a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java +++ b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java @@ -4,6 +4,8 @@ package net.sourceforge.pmd.docs; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; import java.nio.file.FileSystems; @@ -34,6 +36,8 @@ public class RuleSetResolverTest { filterRuleSets(additionalRulesets); + assertFalse(additionalRulesets.isEmpty()); + RuleSetFactory ruleSetFactory = RulesetsFactoryUtils.defaultFactory(); for (String filename : additionalRulesets) { try { @@ -44,6 +48,11 @@ public class RuleSetResolverTest { } } + @Test + public void testAdditionalRulesetPattern() { + assertTrue(GenerateRuleDocsCmd.ADDITIONAL_RULESET_PATTERN.matcher("/home/foo/pmd/pmd-java/src/main/resources/rulesets/java/quickstart.xml").matches()); + } + private void filterRuleSets(List additionalRulesets) { Iterator it = additionalRulesets.iterator(); while (it.hasNext()) { From c31278504c1daad06af61daa62fc0b33652ba7a7 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 21 Jul 2022 10:53:50 +0200 Subject: [PATCH 070/136] Fix test under Windows --- .../java/net/sourceforge/pmd/docs/RuleSetResolverTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java index 7894f8c97f..bafaed3623 100644 --- a/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java +++ b/pmd-doc/src/test/java/net/sourceforge/pmd/docs/RuleSetResolverTest.java @@ -50,7 +50,8 @@ public class RuleSetResolverTest { @Test public void testAdditionalRulesetPattern() { - assertTrue(GenerateRuleDocsCmd.ADDITIONAL_RULESET_PATTERN.matcher("/home/foo/pmd/pmd-java/src/main/resources/rulesets/java/quickstart.xml").matches()); + String filePath = IOUtil.normalizePath("/home/foo/pmd/pmd-java/src/main/resources/rulesets/java/quickstart.xml"); + assertTrue(GenerateRuleDocsCmd.ADDITIONAL_RULESET_PATTERN.matcher(filePath).matches()); } private void filterRuleSets(List additionalRulesets) { From bf5828d7d176d493d165589e497c350257efd748 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Thu, 28 Jul 2022 14:59:08 +0200 Subject: [PATCH 071/136] [doc] Fix jdoc link in release notes --- docs/pages/release_notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b45e8879f0..a41cae583f 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -88,7 +88,7 @@ Being based on a proper Antlr grammar, CPD can: but it is no longer supported with Java 19 Preview. * The interface {% jdoc core::cpd.renderer.CPDRenderer %} is deprecated. For custom CPD renderers the new interface {% jdoc core::cpd.renderer.CPDReportRenderer %} should be used. -* The class {% jdoc test::testframework.TestDescriptor %} is deprecated, replaced with {% jdoc test-schema::testframework.RuleTestDescriptor %}. +* The class {% jdoc test::testframework.TestDescriptor %} is deprecated, replaced with {% jdoc test-schema::test.schema.RuleTestDescriptor %}. * Many methods of {% jdoc test::testframework.RuleTst %} have been deprecated as internal API. #### Experimental APIs From ee38e7d81a0733b18da23a61c9558d90016b602b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 29 Jul 2022 10:11:35 +0200 Subject: [PATCH 072/136] [core] Add "--debug" flag for CPD Fixes #3796 --- docs/pages/pmd/userdocs/cpd/cpd.md | 3 ++ docs/pages/release_notes.md | 6 ++++ .../java/net/sourceforge/pmd/cpd/CPD.java | 16 ++++++++++ .../sourceforge/pmd/cpd/CPDConfiguration.java | 13 ++++++++ .../pmd/cpd/CPDCommandLineInterfaceTest.java | 32 ++++++++++++++++--- 5 files changed, 65 insertions(+), 5 deletions(-) diff --git a/docs/pages/pmd/userdocs/cpd/cpd.md b/docs/pages/pmd/userdocs/cpd/cpd.md index d909eb4f2f..67b81492be 100644 --- a/docs/pages/pmd/userdocs/cpd/cpd.md +++ b/docs/pages/pmd/userdocs/cpd/cpd.md @@ -73,6 +73,9 @@ Novice as much as advanced readers may want to [read on on Refactoring Guru](htt description="Sources code language." default="java" %} + {% include custom/cli_option_row.html options="--debug,--verbose" + description="Debug mode. Prints more log output." + %} {% include custom/cli_option_row.html options="--encoding" description="Character encoding to use when processing files. If not specified, CPD uses the system default encoding." %} diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index a41cae583f..dc7f15c512 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -44,6 +44,7 @@ Being based on a proper Antlr grammar, CPD can: * apex * [#4056](https://github.com/pmd/pmd/pull/4056): \[apex] ApexSOQLInjection: Add support count query * core + * [#3796](https://github.com/pmd/pmd/issues/3796): \[core] CPD should also provide a `--debug` flag * [#4021](https://github.com/pmd/pmd/pull/4021): \[core] CPD: Add total number of tokens to XML reports * [#4031](https://github.com/pmd/pmd/issues/4031): \[core] If report is written to stdout, stdout should not be closed * [#4051](https://github.com/pmd/pmd/issues/4051): \[doc] Additional rulesets are not listed in documentation @@ -66,6 +67,11 @@ Being based on a proper Antlr grammar, CPD can: ### API Changes +#### CPD CLI + +* CPD has a new CLI option `--debug`. This option has the same behavior as in PMD. It enables more verbose + logging output. + #### Rule Test Framework * The module "pmd-test", which contains support classes to write rule tests, now **requires Java 8**. If you depend on diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java index 0ffc7b397d..b7a39e7112 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java @@ -16,6 +16,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.logging.ConsoleHandler; import java.util.logging.Level; import java.util.logging.Logger; @@ -28,6 +29,7 @@ import net.sourceforge.pmd.util.IOUtil; import net.sourceforge.pmd.util.database.DBMSMetadata; import net.sourceforge.pmd.util.database.DBURI; import net.sourceforge.pmd.util.database.SourceObject; +import net.sourceforge.pmd.util.log.ScopedLogHandlersManager; public class CPD { private static final Logger LOGGER = Logger.getLogger(CPD.class.getName()); @@ -54,8 +56,10 @@ public class CPD { } public void go() { + LOGGER.fine("Running match algorithm on " + source.size() + " files..."); matchAlgorithm = new MatchAlgorithm(source, tokens, configuration.getMinimumTileSize(), listener); matchAlgorithm.findMatches(); + LOGGER.fine("Finished: " + matchAlgorithm.getMatches().size() + " duplicates found"); } public Iterator getMatches() { @@ -80,6 +84,7 @@ public class CPD { if (!dir.exists()) { throw new FileNotFoundException("Couldn't find directory " + dir); } + LOGGER.fine("Searching directory " + dir + " for files"); FileFinder finder = new FileFinder(); // TODO - could use SourceFileSelector here add(finder.findFilesFrom(dir, configuration.filenameFilter(), recurse)); @@ -145,6 +150,7 @@ public class CPD { } private void addAndThrowLexicalError(SourceCode sourceCode) throws IOException { + LOGGER.fine("Tokenizing " + sourceCode.getFileName()); configuration.tokenizer().tokenize(sourceCode, tokens); listener.addedFile(1, new File(sourceCode.getFileName())); source.put(sourceCode.getFileName(), sourceCode); @@ -206,6 +212,13 @@ public class CPD { return statusCode; } + final Level logLevel = arguments.isDebug() ? Level.FINER : Level.INFO; + final ScopedLogHandlersManager logHandlerManager = new ScopedLogHandlersManager(logLevel, new ConsoleHandler()); + final Level oldLogLevel = LOGGER.getLevel(); + // Need to do this, since the static logger has already been initialized + // at this point + LOGGER.setLevel(logLevel); + CPD cpd = new CPD(arguments); try { @@ -233,6 +246,9 @@ public class CPD { e.printStackTrace(); LOGGER.severe(CliMessages.errorDetectedMessage(1, CPDCommandLineInterface.PROGRAM_NAME)); statusCode = StatusCode.ERROR; + } finally { + logHandlerManager.close(); + LOGGER.setLevel(oldLogLevel); } return statusCode; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java index abda86e5e0..82906e807b 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPDConfiguration.java @@ -141,6 +141,9 @@ public class CPDConfiguration extends AbstractConfiguration { description = "By default CPD exits with status 4 if code duplications are found. Disable this option with '-failOnViolation false' to exit with 0 instead and just write the report.") private boolean failOnViolation = true; + @Parameter(names = { "--debug", "--verbose" }, description = "Debug mode.") + private boolean debug = false; + // this has to be a public static class, so that JCommander can use it! public static class LanguageConverter implements IStringConverter { @@ -540,4 +543,14 @@ public class CPDConfiguration extends AbstractConfiguration { public void setFailOnViolation(boolean failOnViolation) { this.failOnViolation = failOnViolation; } + + @Override + public boolean isDebug() { + return debug; + } + + @Override + public void setDebug(boolean debug) { + this.debug = debug; + } } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java index ce680e4c5b..9dcabdc35a 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java @@ -4,6 +4,7 @@ package net.sourceforge.pmd.cpd; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -17,9 +18,9 @@ import java.util.Collection; import java.util.List; import java.util.Map; -import org.junit.Assert; import org.junit.Rule; import org.junit.Test; +import org.junit.contrib.java.lang.system.SystemErrRule; import org.junit.contrib.java.lang.system.SystemOutRule; import org.junit.rules.TemporaryFolder; @@ -39,6 +40,10 @@ public class CPDCommandLineInterfaceTest { @Rule public final SystemOutRule log = new SystemOutRule().enableLog().muteForSuccessfulTests(); + + @Rule + public final SystemErrRule errLog = new SystemErrRule().enableLog().muteForSuccessfulTests(); + @Rule public final JavaUtilLoggingRule loggingRule = new JavaUtilLoggingRule(PMD.class.getPackage().getName()).mute(); @Rule @@ -65,8 +70,8 @@ public class CPDCommandLineInterfaceTest { CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", SRC_DIR, "--format", "xml"); final String expectedFilesXml = getExpectedFileEntriesXml(NUMBER_OF_TOKENS.keySet()); - Assert.assertEquals(CPD.StatusCode.OK, statusCode); - Assert.assertEquals("" + "\n" + "" + "\n" + expectedFilesXml + "", log.getLog()); + assertEquals(CPD.StatusCode.OK, statusCode); + assertEquals("" + "\n" + "" + "\n" + expectedFilesXml + "", log.getLog()); } @Test @@ -80,12 +85,29 @@ public class CPDCommandLineInterfaceTest { CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--filelist", filelist.toAbsolutePath().toString(), "--format", "xml", "-failOnViolation", "true"); - Assert.assertEquals(CPD.StatusCode.OK, statusCode); - Assert.assertEquals("" + "\n" + "" + "\n" + expectedFilesXml + "", log.getLog()); + assertEquals(CPD.StatusCode.OK, statusCode); + assertEquals("" + "\n" + "" + "\n" + expectedFilesXml + "", log.getLog()); assertTrue(loggingRule.getLog().contains("Some deprecated options were used on the command-line, including -failOnViolation")); assertTrue(loggingRule.getLog().contains("Consider replacing it with --fail-on-violation")); // only one parameter is logged assertFalse(loggingRule.getLog().contains("Some deprecated options were used on the command-line, including --filelist")); assertFalse(loggingRule.getLog().contains("Consider replacing it with --file-list")); } + + @Test + public void testDebugLogging() { + CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", + SRC_DIR, "--debug"); + assertEquals(CPD.StatusCode.OK, statusCode); + assertTrue(errLog.getLog().contains("Tokenizing ")); // this is a debug logging + } + + @Test + public void testNormalLogging() { + loggingRule.clear(); + CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", + SRC_DIR); + assertEquals(CPD.StatusCode.OK, statusCode); + assertFalse(errLog.getLog().contains("Tokenizing ")); // this is a debug logging + } } From 815242a39af6d9e84eaaf638d5c62d73e1fd541e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 29 Jul 2022 12:12:39 +0200 Subject: [PATCH 073/136] [core] Store languageVersion in DataSource in order to support FileCollector.addFile with language Fixes #3970 --- .../sourceforge/pmd/SourceCodeProcessor.java | 2 +- .../pmd/lang/LanguageVersionDiscoverer.java | 28 +++---------------- .../pmd/lang/document/FileCollector.java | 4 +-- .../pmd/lang/document/NioTextFile.java | 4 +-- .../pmd/lang/document/StringTextFile.java | 7 +++-- .../pmd/processor/PmdRunnable.java | 4 +++ .../internal/FileDataSourceWithLanguage.java | 26 +++++++++++++++++ .../internal/LanguageAwareDataSource.java | 13 +++++++++ .../ReaderDataSourceWithLanguage.java | 26 +++++++++++++++++ .../net/sourceforge/pmd/PmdAnalysisTest.java | 23 +++++++++++++++ .../pmd/lang/document/SimpleTestTextFile.java | 17 +++++++++++ 11 files changed, 121 insertions(+), 33 deletions(-) create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java create mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java create mode 100644 pmd-core/src/test/java/net/sourceforge/pmd/lang/document/SimpleTestTextFile.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java b/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java index f52b0f2a2a..7d1838c297 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/SourceCodeProcessor.java @@ -222,7 +222,7 @@ public class SourceCodeProcessor { ctx.setLanguageVersion(forceLanguage); } else { // otherwise determine by file extension - LanguageVersion languageVersion = configuration.getLanguageVersionOfFile(ctx.getSourceCodeFile().toString()); + LanguageVersion languageVersion = configuration.getLanguageVersionOfFile(ctx.getSourceCodeFilename()); ctx.setLanguageVersion(languageVersion); } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java index 27c72898e1..4e22b03a4f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java @@ -5,8 +5,6 @@ package net.sourceforge.pmd.lang; import java.io.File; -import java.io.IOException; -import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -14,7 +12,6 @@ import java.util.Objects; import org.apache.commons.lang3.StringUtils; -import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.internal.util.AssertionUtil; /** @@ -27,8 +24,6 @@ public class LanguageVersionDiscoverer { private LanguageVersion forcedVersion; - private Map forcedVersionByFile = new HashMap<>(); - public LanguageVersionDiscoverer() { this(null); } @@ -102,13 +97,10 @@ public class LanguageVersionDiscoverer { * file. */ public LanguageVersion getDefaultLanguageVersionForFile(String fileName) { - LanguageVersion languageVersion = forcedVersionByFile.get(fileName); - - if (languageVersion == null) { - List languages = getLanguagesForFile(fileName); - if (!languages.isEmpty()) { - languageVersion = getDefaultLanguageVersion(languages.get(0)); - } + List languages = getLanguagesForFile(fileName); + LanguageVersion languageVersion = null; + if (!languages.isEmpty()) { + languageVersion = getDefaultLanguageVersion(languages.get(0)); } return languageVersion; } @@ -150,16 +142,4 @@ public class LanguageVersionDiscoverer { } - - @InternalApi - @Deprecated - public void recordLanguageVersionForFile(Path file, LanguageVersion languageVersion) { - String fileName; - try { - fileName = file.toRealPath().toString(); - } catch (IOException e) { - fileName = file.toAbsolutePath().toString(); - } - forcedVersionByFile.put(fileName, languageVersion); - } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java index 51d84106c4..8db97175c3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/FileCollector.java @@ -155,10 +155,8 @@ public final class FileCollector implements AutoCloseable { reporter.error("Not a regular file {0}", file); return false; } - LanguageVersion languageVersion = discoverer.getDefaultLanguageVersion(language); - NioTextFile nioTextFile = new NioTextFile(file, charset, languageVersion, getDisplayName(file)); + NioTextFile nioTextFile = new NioTextFile(file, charset, discoverer.getDefaultLanguageVersion(language), getDisplayName(file)); addFileImpl(nioTextFile); - discoverer.recordLanguageVersionForFile(file, languageVersion); return true; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java index c7d77e1de8..8fc154967a 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java @@ -17,7 +17,7 @@ import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.util.IOUtil; import net.sourceforge.pmd.util.datasource.DataSource; -import net.sourceforge.pmd.util.datasource.FileDataSource; +import net.sourceforge.pmd.util.datasource.internal.FileDataSourceWithLanguage; /** * A {@link TextFile} backed by a file in some {@link FileSystem}. @@ -73,7 +73,7 @@ class NioTextFile implements TextFile { @Override public DataSource toDataSourceCompat() { - return new FileDataSource(path.toFile()); + return new FileDataSourceWithLanguage(path.toFile(), languageVersion); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java index 598823e555..db2f537eee 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java @@ -11,7 +11,7 @@ import net.sourceforge.pmd.annotation.Experimental; import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.util.datasource.DataSource; -import net.sourceforge.pmd.util.datasource.ReaderDataSource; +import net.sourceforge.pmd.util.datasource.internal.ReaderDataSourceWithLanguage; /** * Read-only view on a string. @@ -64,9 +64,10 @@ class StringTextFile implements TextFile { @Override public DataSource toDataSourceCompat() { - return new ReaderDataSource( + return new ReaderDataSourceWithLanguage( new StringReader(content), - pathId + pathId, + languageVersion ); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java b/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java index da91fbd95a..e4b1d75de6 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/processor/PmdRunnable.java @@ -21,6 +21,7 @@ import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.benchmark.TimeTracker; import net.sourceforge.pmd.renderers.Renderer; import net.sourceforge.pmd.util.datasource.DataSource; +import net.sourceforge.pmd.util.datasource.internal.LanguageAwareDataSource; /** * @@ -82,6 +83,9 @@ public class PmdRunnable implements Callable { try (InputStream stream = new BufferedInputStream(dataSource.getInputStream())) { tc.ruleContext.setLanguageVersion(null); + if (dataSource instanceof LanguageAwareDataSource) { + tc.ruleContext.setLanguageVersion(((LanguageAwareDataSource) dataSource).getLanguageVersion()); + } sourceCodeProcessor.processSourceCode(stream, tc.ruleSets, tc.ruleContext); } catch (PMDException pmde) { addError(report, pmde, "Error while processing file: " + fileName); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java new file mode 100644 index 0000000000..7ca0e1c8a4 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java @@ -0,0 +1,26 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.datasource.internal; + +import java.io.File; + +import net.sourceforge.pmd.annotation.InternalApi; +import net.sourceforge.pmd.lang.LanguageVersion; +import net.sourceforge.pmd.util.datasource.FileDataSource; + +@InternalApi +public class FileDataSourceWithLanguage extends FileDataSource implements LanguageAwareDataSource { + private final LanguageVersion languageVersion; + + public FileDataSourceWithLanguage(File file, LanguageVersion languageVersion) { + super(file); + this.languageVersion = languageVersion; + } + + @Override + public LanguageVersion getLanguageVersion() { + return languageVersion; + } +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java new file mode 100644 index 0000000000..69dbf67059 --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java @@ -0,0 +1,13 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.datasource.internal; + +import net.sourceforge.pmd.annotation.InternalApi; +import net.sourceforge.pmd.lang.LanguageVersion; + +@InternalApi +public interface LanguageAwareDataSource { + LanguageVersion getLanguageVersion(); +} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java new file mode 100644 index 0000000000..e96429e65d --- /dev/null +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java @@ -0,0 +1,26 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.util.datasource.internal; + +import java.io.Reader; + +import net.sourceforge.pmd.annotation.InternalApi; +import net.sourceforge.pmd.lang.LanguageVersion; +import net.sourceforge.pmd.util.datasource.ReaderDataSource; + +@InternalApi +public class ReaderDataSourceWithLanguage extends ReaderDataSource implements LanguageAwareDataSource { + private final LanguageVersion languageVersion; + + public ReaderDataSourceWithLanguage(Reader reader, String dataSourceName, LanguageVersion languageVersion) { + super(reader, dataSourceName); + this.languageVersion = languageVersion; + } + + @Override + public LanguageVersion getLanguageVersion() { + return languageVersion; + } +} diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java index 797f4e0569..a1f2f0f156 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java @@ -24,6 +24,7 @@ import org.mockito.ArgumentMatchers; import net.sourceforge.pmd.lang.Dummy2LanguageModule; import net.sourceforge.pmd.lang.Language; import net.sourceforge.pmd.lang.ast.Node; +import net.sourceforge.pmd.lang.document.SimpleTestTextFile; import net.sourceforge.pmd.lang.rule.AbstractRule; import net.sourceforge.pmd.renderers.Renderer; @@ -90,6 +91,28 @@ public class PmdAnalysisTest { pmd.addRuleSet(ruleset); pmd.files().addFile(Paths.get("src", "test", "resources", "sample-source", "dummy", "foo.txt"), language); Report report = pmd.performAnalysisAndCollectReport(); + for (Report.ProcessingError error : report.getProcessingErrors()) { + System.out.println("error = " + error.getMsg() + ": " + error.getDetail()); + } + Assert.assertEquals(0, report.getProcessingErrors().size()); + Assert.assertEquals(1, report.getViolations().size()); + } + } + + @Test + public void testTextFileWithSpecificLanguage() { + final Language language = Dummy2LanguageModule.getInstance(); + PMDConfiguration config = new PMDConfiguration(); + config.setIgnoreIncrementalAnalysis(true); + RuleSet ruleset = RuleSet.forSingleRule(new TestRule()); + + try (PmdAnalysis pmd = PmdAnalysis.create(config)) { + pmd.addRuleSet(ruleset); + pmd.files().addFile(new SimpleTestTextFile("test content foo", "foo.txt", "foo.txt", language.getDefaultVersion())); + Report report = pmd.performAnalysisAndCollectReport(); + for (Report.ProcessingError error : report.getProcessingErrors()) { + System.out.println("error = " + error.getMsg() + ": " + error.getDetail()); + } Assert.assertEquals(0, report.getProcessingErrors().size()); Assert.assertEquals(1, report.getViolations().size()); } diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/SimpleTestTextFile.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/SimpleTestTextFile.java new file mode 100644 index 0000000000..6f66347142 --- /dev/null +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/document/SimpleTestTextFile.java @@ -0,0 +1,17 @@ +/* + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.document; + +import net.sourceforge.pmd.lang.LanguageVersion; + +/** + * Makes {@link StringTextFile} publicly available for unit tests. + */ +public class SimpleTestTextFile extends StringTextFile { + + public SimpleTestTextFile(String content, String pathId, String displayName, LanguageVersion languageVersion) { + super(content, pathId, displayName, languageVersion); + } +} From a36032b2bfab942ba8949808de217aa29950baca Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 29 Jul 2022 12:33:43 +0200 Subject: [PATCH 074/136] Fixups from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: ClΓ©ment Fournier --- .../sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java index 9dcabdc35a..a01b355b7d 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java @@ -4,6 +4,9 @@ package net.sourceforge.pmd.cpd; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.not; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -99,7 +102,7 @@ public class CPDCommandLineInterfaceTest { CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", SRC_DIR, "--debug"); assertEquals(CPD.StatusCode.OK, statusCode); - assertTrue(errLog.getLog().contains("Tokenizing ")); // this is a debug logging + assertThat(errLog.getLog(), containsString("Tokenizing ")); // this is a debug logging } @Test @@ -108,6 +111,6 @@ public class CPDCommandLineInterfaceTest { CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", SRC_DIR); assertEquals(CPD.StatusCode.OK, statusCode); - assertFalse(errLog.getLog().contains("Tokenizing ")); // this is a debug logging + assertThat(errLog.getLog(), not(containsString("Tokenizing "))); // this is a debug logging } } From ca87fe51e37676d613f68c1a9d9d7bd222dbee69 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 29 Jul 2022 15:24:30 +0200 Subject: [PATCH 075/136] Fixups from review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: ClΓ©ment Fournier --- .../pmd/lang/document/NioTextFile.java | 5 +-- .../pmd/lang/document/StringTextFile.java | 7 ++-- .../internal/FileDataSourceWithLanguage.java | 26 --------------- .../internal/LanguageAwareDataSource.java | 33 +++++++++++++++++-- .../ReaderDataSourceWithLanguage.java | 26 --------------- 5 files changed, 38 insertions(+), 59 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java index 8fc154967a..16ab46f93f 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/NioTextFile.java @@ -17,7 +17,8 @@ import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.util.IOUtil; import net.sourceforge.pmd.util.datasource.DataSource; -import net.sourceforge.pmd.util.datasource.internal.FileDataSourceWithLanguage; +import net.sourceforge.pmd.util.datasource.FileDataSource; +import net.sourceforge.pmd.util.datasource.internal.LanguageAwareDataSource; /** * A {@link TextFile} backed by a file in some {@link FileSystem}. @@ -73,7 +74,7 @@ class NioTextFile implements TextFile { @Override public DataSource toDataSourceCompat() { - return new FileDataSourceWithLanguage(path.toFile(), languageVersion); + return new LanguageAwareDataSource(new FileDataSource(path.toFile()), languageVersion); } @Override diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java index db2f537eee..c4d286d895 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/StringTextFile.java @@ -11,7 +11,8 @@ import net.sourceforge.pmd.annotation.Experimental; import net.sourceforge.pmd.internal.util.AssertionUtil; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.util.datasource.DataSource; -import net.sourceforge.pmd.util.datasource.internal.ReaderDataSourceWithLanguage; +import net.sourceforge.pmd.util.datasource.ReaderDataSource; +import net.sourceforge.pmd.util.datasource.internal.LanguageAwareDataSource; /** * Read-only view on a string. @@ -64,9 +65,9 @@ class StringTextFile implements TextFile { @Override public DataSource toDataSourceCompat() { - return new ReaderDataSourceWithLanguage( + return new LanguageAwareDataSource(new ReaderDataSource( new StringReader(content), - pathId, + pathId), languageVersion ); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java deleted file mode 100644 index 7ca0e1c8a4..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/FileDataSourceWithLanguage.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.util.datasource.internal; - -import java.io.File; - -import net.sourceforge.pmd.annotation.InternalApi; -import net.sourceforge.pmd.lang.LanguageVersion; -import net.sourceforge.pmd.util.datasource.FileDataSource; - -@InternalApi -public class FileDataSourceWithLanguage extends FileDataSource implements LanguageAwareDataSource { - private final LanguageVersion languageVersion; - - public FileDataSourceWithLanguage(File file, LanguageVersion languageVersion) { - super(file); - this.languageVersion = languageVersion; - } - - @Override - public LanguageVersion getLanguageVersion() { - return languageVersion; - } -} diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java index 69dbf67059..407001e102 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java @@ -4,10 +4,39 @@ package net.sourceforge.pmd.util.datasource.internal; +import java.io.IOException; +import java.io.InputStream; + import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.lang.LanguageVersion; +import net.sourceforge.pmd.util.datasource.DataSource; @InternalApi -public interface LanguageAwareDataSource { - LanguageVersion getLanguageVersion(); +public class LanguageAwareDataSource implements DataSource { + private final DataSource base; // delegate DataSource methods to this + private final LanguageVersion version; + + public LanguageAwareDataSource(DataSource base, LanguageVersion version) { + this.base = base; + this.version = version; + } + + public LanguageVersion getLanguageVersion() { + return version; + } + + @Override + public InputStream getInputStream() throws IOException { + return base.getInputStream(); + } + + @Override + public String getNiceFileName(boolean shortNames, String inputFileName) { + return base.getNiceFileName(shortNames, inputFileName); + } + + @Override + public void close() throws IOException { + base.close(); + } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java deleted file mode 100644 index e96429e65d..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/ReaderDataSourceWithLanguage.java +++ /dev/null @@ -1,26 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.util.datasource.internal; - -import java.io.Reader; - -import net.sourceforge.pmd.annotation.InternalApi; -import net.sourceforge.pmd.lang.LanguageVersion; -import net.sourceforge.pmd.util.datasource.ReaderDataSource; - -@InternalApi -public class ReaderDataSourceWithLanguage extends ReaderDataSource implements LanguageAwareDataSource { - private final LanguageVersion languageVersion; - - public ReaderDataSourceWithLanguage(Reader reader, String dataSourceName, LanguageVersion languageVersion) { - super(reader, dataSourceName); - this.languageVersion = languageVersion; - } - - @Override - public LanguageVersion getLanguageVersion() { - return languageVersion; - } -} From 6128631acb420119b65126273db3a5013ff09729 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 09:59:57 +0200 Subject: [PATCH 076/136] [doc] Add SPONSORS.md --- SPONSORS.md | 10 ++++++++++ docs/pages/release_notes.md | 6 ++++++ 2 files changed, 16 insertions(+) create mode 100644 SPONSORS.md diff --git a/SPONSORS.md b/SPONSORS.md new file mode 100644 index 0000000000..4bbe1c7b55 --- /dev/null +++ b/SPONSORS.md @@ -0,0 +1,10 @@ +# PMD's sponsors + +Many thanks to all our sponsors: + +* [Matt Hargett](https://github.com/matthargett) (@matthargett) + +If you also want to sponsor PMD, you have two options: + +* [Sponsor @pmd on GitHub Sponsors](https://github.com/sponsors/pmd) +* [PMD - Open Collective](https://opencollective.com/pmd) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index a41cae583f..7a305420e2 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -112,6 +112,12 @@ You can identify them with the `@InternalApi` annotation. You'll also get a depr * {%jdoc !!core::cpd.CPDConfiguration#getCPDRendererFromString(java.lang.String,java.lang.String) %} * {%jdoc core::cpd.renderer.CPDRendererAdapter %} +### Financial Contributions + +Many thanks to our sponsors: + +* [Matt Hargett](https://github.com/matthargett) (@matthargett) + ### External Contributions * [#3984](https://github.com/pmd/pmd/pull/3984): \[java] Fix AddEmptyString false-negative issue - [@LiGaOg](https://github.com/LiGaOg) * [#3988](https://github.com/pmd/pmd/pull/3988): \[java] Modify WhileLoopWithLiteralBoolean to meet the missing case #3455 - [@VoidxHoshi](https://github.com/VoidxHoshi) From b35454a484032e6e4477b11beaec482d88e935fa Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 10:34:55 +0200 Subject: [PATCH 077/136] [core] Fix CPD logging unit tests --- .../java/net/sourceforge/pmd/cpd/CPD.java | 5 ++-- .../pmd/cpd/CPDCommandLineInterfaceTest.java | 23 +++++++++++++++---- .../pmd/cpd/CPDCommandLineInterfaceTest.java | 3 ++- .../sourceforge/pmd/cli/BaseCPDCLITest.java | 15 +++++++++++- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java index 9840bb7d58..fb8c43b39d 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/CPD.java @@ -218,9 +218,10 @@ public class CPD { // otherwise just use whatever is in conf/simplelogger.properties which happens automatically if (arguments.isDebug()) { Slf4jSimpleConfiguration.reconfigureDefaultLogLevel(Level.TRACE); - // need to reload the logger with the new configuration - log = LoggerFactory.getLogger(CPD.class); } + // always need to reload the logger with the new/changed configuration + // unit tests might reset the logging configuration + log = LoggerFactory.getLogger(CPD.class); // TODO CLI errors should also be reported through this // TODO this should not use the logger as backend, otherwise without diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java index 3b4ada5df7..8ed63b71ed 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java @@ -20,10 +20,13 @@ import java.util.Collection; import java.util.List; import java.util.Map; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import net.sourceforge.pmd.internal.Slf4jSimpleConfiguration; + import com.github.stefanbirkner.systemlambda.SystemLambda; import com.google.common.collect.ImmutableMap; @@ -39,6 +42,13 @@ class CPDCommandLineInterfaceTest { @TempDir private Path tempDir; + @AfterEach + void resetLogging() { + // reset logging in case "--debug" changed the logging properties + // See also Slf4jSimpleConfigurationForAnt + Slf4jSimpleConfiguration.reconfigureDefaultLogLevel(null); + } + @BeforeEach void setup() { System.setProperty(CPDCommandLineInterface.NO_EXIT_AFTER_RUN, "true"); @@ -98,12 +108,15 @@ class CPDCommandLineInterfaceTest { @Test void testDebugLogging() throws Exception { - String stderr = SystemLambda.tapSystemErr(() -> { - CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", - SRC_DIR, "--debug"); - assertEquals(CPD.StatusCode.OK, statusCode); + // restoring system properties: --debug might change logging properties + SystemLambda.restoreSystemProperties(() -> { + String stderr = SystemLambda.tapSystemErr(() -> { + CPD.StatusCode statusCode = CPD.runCpd("--minimum-tokens", "340", "--language", "java", "--files", + SRC_DIR, "--debug"); + assertEquals(CPD.StatusCode.OK, statusCode); + }); + assertThat(stderr, containsString("Tokenizing ")); // this is a debug logging }); - assertThat(stderr, containsString("Tokenizing ")); // this is a debug logging } @Test diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java index 3d7c2f8b54..0da64dd11f 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/cpd/CPDCommandLineInterfaceTest.java @@ -91,7 +91,8 @@ public class CPDCommandLineInterfaceTest extends BaseCPDCLITest { public void testBrokenAndValidFile() { String out = runTest(CPD.StatusCode.DUPLICATE_CODE_FOUND, "--minimum-tokens", "10", "--language", "java", "--files", "src/test/resources/net/sourceforge/pmd/cpd/badandgood/", "--format", "text", "--skip-lexical-errors"); - assertThat(out, containsPattern("Skipping .*?BadFile\\.java\\. Reason: Lexical error in file")); + String stderr = getStderr(); + assertThat(stderr, containsPattern("Skipping .*?BadFile\\.java\\. Reason: Lexical error in file")); assertThat(out, containsString("Found a 5 line (13 tokens) duplication")); } diff --git a/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCPDCLITest.java b/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCPDCLITest.java index 3380d7acc4..a5ef1450a7 100644 --- a/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCPDCLITest.java +++ b/pmd-test/src/main/java/net/sourceforge/pmd/cli/BaseCPDCLITest.java @@ -17,6 +17,9 @@ import net.sourceforge.pmd.cpd.CPDCommandLineInterface; public abstract class BaseCPDCLITest { private ByteArrayOutputStream bufferStdout; + + private ByteArrayOutputStream bufferStderr; + private PrintStream originalStdout; private PrintStream originalStderr; @@ -26,7 +29,9 @@ public abstract class BaseCPDCLITest { originalStderr = System.err; bufferStdout = new ByteArrayOutputStream(); System.setOut(new PrintStream(bufferStdout, false, "UTF-8")); - System.setErr(System.out); + + bufferStderr = new ByteArrayOutputStream(); + System.setErr(new PrintStream(bufferStderr, false, "UTF-8")); } @After @@ -56,6 +61,14 @@ public abstract class BaseCPDCLITest { CPD.main(args); } + protected String getStderr() { + try { + return bufferStderr.toString("UTF-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException(e); + } + } + protected String runTest(CPD.StatusCode expectedStatusCode, String... args) { CPD.StatusCode statusCode = CPD.runCpd(args); Assert.assertEquals("Unexpected status code", expectedStatusCode, statusCode); From afaeeb9684bdf69dd78660f49a7a05bcaab9bd4d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 11:15:56 +0200 Subject: [PATCH 078/136] [core] Fix old ruleset 35.xml --- pmd-core/src/main/resources/rulesets/releases/35.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pmd-core/src/main/resources/rulesets/releases/35.xml b/pmd-core/src/main/resources/rulesets/releases/35.xml index debffe92c2..70263d0435 100644 --- a/pmd-core/src/main/resources/rulesets/releases/35.xml +++ b/pmd-core/src/main/resources/rulesets/releases/35.xml @@ -25,7 +25,9 @@ This ruleset contains links to rules that are new in PMD v3.5 + From 143545c39001d5ec666677ada10004d90b3ea56d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 11:23:41 +0200 Subject: [PATCH 079/136] Prepare pmd release 6.48.0 --- docs/_config.yml | 2 +- docs/pages/next_major_development.md | 53 ++++++++++++++++++++++++++++ docs/pages/release_notes.md | 5 +++ 3 files changed, 59 insertions(+), 1 deletion(-) diff --git a/docs/_config.yml b/docs/_config.yml index a75b2e740a..5ccd6a03b2 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,7 +1,7 @@ repository: pmd/pmd pmd: - version: 6.48.0-SNAPSHOT + version: 6.48.0 previous_version: 6.47.0 date: 30-July-2022 release_type: minor diff --git a/docs/pages/next_major_development.md b/docs/pages/next_major_development.md index 6c53ffc407..5cc83709eb 100644 --- a/docs/pages/next_major_development.md +++ b/docs/pages/next_major_development.md @@ -125,6 +125,59 @@ the breaking API changes will be performed in 7.0.0. an API is tagged as `@Deprecated` or not in the latest minor release. During the development of 7.0.0, we may decide to remove some APIs that were not tagged as deprecated, though we'll try to avoid it." %} +#### 6.48.0 + +##### CPD CLI + +* CPD has a new CLI option `--debug`. This option has the same behavior as in PMD. It enables more verbose + logging output. + +##### Rule Test Framework + +* The module "pmd-test", which contains support classes to write rule tests, now **requires Java 8**. If you depend on + this module for testing your own custom rules, you'll need to make sure to use at least Java 8. +* The new module "pmd-test-schema" contains now the XSD schema and the code to parse the rule test XML files. The + schema has been extracted in order to easily share it with other tools like the Rule Designer or IDE plugins. +* Test schema changes: + * The attribute `isRegressionTest` of `test-code` is deprecated. The new + attribute `disabled` should be used instead for defining whether a rule test should be skipped or not. + * The attributes `reinitializeRule` and `useAuxClasspath` of `test-code` are deprecated and assumed true. + They will not be replaced. + * The new attribute `focused` of `test-code` allows disabling all tests except the focused one temporarily. +* More information about the rule test framework can be found in the documentation: + [Testing your rules](pmd_userdocs_extending_testing.html) + +##### Deprecated API + +* The experimental Java AST class {% jdoc java::lang.java.ast.ASTGuardedPattern %} has been deprecated and + will be removed. It was introduced for Java 17 and Java 18 Preview as part of pattern matching for switch, + but it is no longer supported with Java 19 Preview. +* The interface {% jdoc core::cpd.renderer.CPDRenderer %} is deprecated. For custom CPD renderers + the new interface {% jdoc core::cpd.renderer.CPDReportRenderer %} should be used. +* The class {% jdoc test::testframework.TestDescriptor %} is deprecated, replaced with {% jdoc test-schema::test.schema.RuleTestDescriptor %}. +* Many methods of {% jdoc test::testframework.RuleTst %} have been deprecated as internal API. + +##### Experimental APIs + +* To support the Java preview language features "Pattern Matching for Switch" and "Record Patterns", the following + AST nodes have been introduced as experimental: + * {% jdoc java::lang.java.ast.ASTSwitchGuard %} + * {% jdoc java::lang.java.ast.ASTRecordPattern %} + * {% jdoc java::lang.java.ast.ASTComponentPatternList %} + +##### Internal API + +Those APIs are not intended to be used by clients, and will be hidden or removed with PMD 7.0.0. +You can identify them with the `@InternalApi` annotation. You'll also get a deprecation warning. + +* {%jdoc !!core::cpd.CPDConfiguration#setRenderer(net.sourceforge.pmd.cpd.Renderer) %} +* {%jdoc !!core::cpd.CPDConfiguration#setCPDRenderer(net.sourceforge.pmd.cpd.renderer.CPDRenderer) %} +* {%jdoc !!core::cpd.CPDConfiguration#getRenderer() %} +* {%jdoc !!core::cpd.CPDConfiguration#getCPDRenderer() %} +* {%jdoc !!core::cpd.CPDConfiguration#getRendererFromString(java.lang.String,java.lang.String) %} +* {%jdoc !!core::cpd.CPDConfiguration#getCPDRendererFromString(java.lang.String,java.lang.String) %} +* {%jdoc core::cpd.renderer.CPDRendererAdapter %} + #### 6.47.0 No changes. diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index e809b82300..eeabe3a7dc 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -134,5 +134,10 @@ Many thanks to our sponsors: * [#4056](https://github.com/pmd/pmd/pull/4056): \[apex] ApexSOQLInjection: Add support count query - [@gwilymatgearset](https://github.com/gwilymatgearset) * [#4061](https://github.com/pmd/pmd/pull/4061): \[lua] Fix several related Lua parsing issues found when using CPD - [@matthargett](https://github.com/matthargett) +### Stats +* 102 commits +* 26 closed tickets & PRs +* Days since last release: 35 + {% endtocmaker %} From 6aa2b47f2ed64c2971dd86a44bb4fb62456e728c Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 11:35:53 +0200 Subject: [PATCH 080/136] [maven-release-plugin] prepare release pmd_releases/6.48.0 --- pmd-apex-jorje/pom.xml | 2 +- pmd-apex/pom.xml | 2 +- pmd-core/pom.xml | 2 +- pmd-cpp/pom.xml | 2 +- pmd-cs/pom.xml | 2 +- pmd-dart/pom.xml | 2 +- pmd-dist/pom.xml | 2 +- pmd-doc/pom.xml | 2 +- pmd-fortran/pom.xml | 2 +- pmd-gherkin/pom.xml | 2 +- pmd-go/pom.xml | 2 +- pmd-groovy/pom.xml | 2 +- pmd-html/pom.xml | 2 +- pmd-java/pom.xml | 2 +- pmd-java8/pom.xml | 2 +- pmd-javascript/pom.xml | 2 +- pmd-jsp/pom.xml | 2 +- pmd-kotlin/pom.xml | 2 +- pmd-lang-test/pom.xml | 2 +- pmd-lua/pom.xml | 2 +- pmd-matlab/pom.xml | 2 +- pmd-modelica/pom.xml | 2 +- pmd-objectivec/pom.xml | 2 +- pmd-perl/pom.xml | 2 +- pmd-php/pom.xml | 2 +- pmd-plsql/pom.xml | 2 +- pmd-python/pom.xml | 2 +- pmd-ruby/pom.xml | 2 +- pmd-scala-modules/pmd-scala-common/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.12/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.13/pom.xml | 2 +- pmd-scala/pom.xml | 2 +- pmd-swift/pom.xml | 2 +- pmd-test-schema/pom.xml | 6 ++---- pmd-test/pom.xml | 2 +- pmd-visualforce/pom.xml | 2 +- pmd-vm/pom.xml | 2 +- pmd-xml/pom.xml | 2 +- pom.xml | 8 ++++---- 39 files changed, 43 insertions(+), 45 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index c368f726dc..c605e61a9d 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-apex/pom.xml b/pmd-apex/pom.xml index 5a5f208f72..a661b0e3c8 100644 --- a/pmd-apex/pom.xml +++ b/pmd-apex/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index d7bf9e1b27..4ef2c6c1c1 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-cpp/pom.xml b/pmd-cpp/pom.xml index cfbe955501..839b79e061 100644 --- a/pmd-cpp/pom.xml +++ b/pmd-cpp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-cs/pom.xml b/pmd-cs/pom.xml index 36a4e08e0a..b6169ce0f5 100644 --- a/pmd-cs/pom.xml +++ b/pmd-cs/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-dart/pom.xml b/pmd-dart/pom.xml index e3c472c466..4b29f14fbc 100644 --- a/pmd-dart/pom.xml +++ b/pmd-dart/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-dist/pom.xml b/pmd-dist/pom.xml index c41b723273..0624ab525a 100644 --- a/pmd-dist/pom.xml +++ b/pmd-dist/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-doc/pom.xml b/pmd-doc/pom.xml index f35338c8c1..1dd433f596 100644 --- a/pmd-doc/pom.xml +++ b/pmd-doc/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-fortran/pom.xml b/pmd-fortran/pom.xml index 55afe0f0ca..c26c301c30 100644 --- a/pmd-fortran/pom.xml +++ b/pmd-fortran/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-gherkin/pom.xml b/pmd-gherkin/pom.xml index e49be3647b..20eda17f8a 100644 --- a/pmd-gherkin/pom.xml +++ b/pmd-gherkin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-go/pom.xml b/pmd-go/pom.xml index 9c7c525e53..febadeb629 100644 --- a/pmd-go/pom.xml +++ b/pmd-go/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-groovy/pom.xml b/pmd-groovy/pom.xml index db176baebe..5fea6def0c 100644 --- a/pmd-groovy/pom.xml +++ b/pmd-groovy/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-html/pom.xml b/pmd-html/pom.xml index db21698140..381b86a9a0 100644 --- a/pmd-html/pom.xml +++ b/pmd-html/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-java/pom.xml b/pmd-java/pom.xml index ddd74247e6..ac5a921e0e 100644 --- a/pmd-java/pom.xml +++ b/pmd-java/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-java8/pom.xml b/pmd-java8/pom.xml index a122fd7a41..47a06c8528 100644 --- a/pmd-java8/pom.xml +++ b/pmd-java8/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-javascript/pom.xml b/pmd-javascript/pom.xml index 1fb34e70f5..844634afde 100644 --- a/pmd-javascript/pom.xml +++ b/pmd-javascript/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-jsp/pom.xml b/pmd-jsp/pom.xml index 2689d0b071..c027744cf0 100644 --- a/pmd-jsp/pom.xml +++ b/pmd-jsp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-kotlin/pom.xml b/pmd-kotlin/pom.xml index 4e226a123d..4e58d15787 100644 --- a/pmd-kotlin/pom.xml +++ b/pmd-kotlin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-lang-test/pom.xml b/pmd-lang-test/pom.xml index b8c6d65a03..1d499c152a 100644 --- a/pmd-lang-test/pom.xml +++ b/pmd-lang-test/pom.xml @@ -12,7 +12,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-lua/pom.xml b/pmd-lua/pom.xml index 180aa92f43..b5505b035e 100644 --- a/pmd-lua/pom.xml +++ b/pmd-lua/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-matlab/pom.xml b/pmd-matlab/pom.xml index af78df7ea9..30e061f46b 100644 --- a/pmd-matlab/pom.xml +++ b/pmd-matlab/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-modelica/pom.xml b/pmd-modelica/pom.xml index 72480f6f5a..48bfa8a679 100644 --- a/pmd-modelica/pom.xml +++ b/pmd-modelica/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-objectivec/pom.xml b/pmd-objectivec/pom.xml index fca206b41e..93a391f0f6 100644 --- a/pmd-objectivec/pom.xml +++ b/pmd-objectivec/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-perl/pom.xml b/pmd-perl/pom.xml index a3a95bb798..cf0886bd93 100644 --- a/pmd-perl/pom.xml +++ b/pmd-perl/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-php/pom.xml b/pmd-php/pom.xml index 029be6c047..6f772c36bb 100644 --- a/pmd-php/pom.xml +++ b/pmd-php/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-plsql/pom.xml b/pmd-plsql/pom.xml index c217149637..d3bbc1bc3b 100644 --- a/pmd-plsql/pom.xml +++ b/pmd-plsql/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-python/pom.xml b/pmd-python/pom.xml index 7a4c58569d..ec6c5e4c30 100644 --- a/pmd-python/pom.xml +++ b/pmd-python/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-ruby/pom.xml b/pmd-ruby/pom.xml index 6631300aa5..532e67da47 100644 --- a/pmd-ruby/pom.xml +++ b/pmd-ruby/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-scala-modules/pmd-scala-common/pom.xml b/pmd-scala-modules/pmd-scala-common/pom.xml index 5029a7d0a3..b303122d0c 100644 --- a/pmd-scala-modules/pmd-scala-common/pom.xml +++ b/pmd-scala-modules/pmd-scala-common/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../../pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.12/pom.xml b/pmd-scala-modules/pmd-scala_2.12/pom.xml index 2de4376841..eadca460a0 100644 --- a/pmd-scala-modules/pmd-scala_2.12/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.12/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.48.0-SNAPSHOT + 6.48.0 ../pmd-scala-common/pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.13/pom.xml b/pmd-scala-modules/pmd-scala_2.13/pom.xml index a7e7e17bed..1468d708b5 100644 --- a/pmd-scala-modules/pmd-scala_2.13/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.13/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.48.0-SNAPSHOT + 6.48.0 ../pmd-scala-common/pom.xml diff --git a/pmd-scala/pom.xml b/pmd-scala/pom.xml index 513cbc7fda..bdd29afbab 100644 --- a/pmd-scala/pom.xml +++ b/pmd-scala/pom.xml @@ -9,7 +9,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-swift/pom.xml b/pmd-swift/pom.xml index 9a600748c1..f19edbaa50 100644 --- a/pmd-swift/pom.xml +++ b/pmd-swift/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-test-schema/pom.xml b/pmd-test-schema/pom.xml index e1dacde506..2f15a89727 100644 --- a/pmd-test-schema/pom.xml +++ b/pmd-test-schema/pom.xml @@ -1,7 +1,5 @@ - + 4.0.0 pmd-test-schema PMD Test Schema @@ -13,7 +11,7 @@ pmd net.sourceforge.pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-test/pom.xml b/pmd-test/pom.xml index a161cf6458..594a0619fa 100644 --- a/pmd-test/pom.xml +++ b/pmd-test/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-visualforce/pom.xml b/pmd-visualforce/pom.xml index 6285b89b29..de6d255a9a 100644 --- a/pmd-visualforce/pom.xml +++ b/pmd-visualforce/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-vm/pom.xml b/pmd-vm/pom.xml index 238203f2f0..3e1e175787 100644 --- a/pmd-vm/pom.xml +++ b/pmd-vm/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pmd-xml/pom.xml b/pmd-xml/pom.xml index c3ade37bbd..58804856eb 100644 --- a/pmd-xml/pom.xml +++ b/pmd-xml/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 ../pom.xml diff --git a/pom.xml b/pom.xml index 065b719b70..91a7b0c728 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 net.sourceforge.pmd pmd - 6.48.0-SNAPSHOT + 6.48.0 pom PMD @@ -55,7 +55,7 @@ scm:git:git://github.com/pmd/pmd.git scm:git:ssh://git@github.com/pmd/pmd.git https://github.com/pmd/pmd - HEAD + pmd_releases/6.48.0 @@ -76,7 +76,7 @@ - 2022-06-25T07:30:42Z + 2022-07-30T09:23:49Z 7 @@ -104,7 +104,7 @@ https://pmd.github.io/pmd -Xmx512m -Dfile.encoding=${project.build.sourceEncoding} ${extraArgLine} - + 18 From 995d00fde3a692905f0d60759e90d05695bdb19c Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 11:35:57 +0200 Subject: [PATCH 081/136] [maven-release-plugin] prepare for next development iteration --- pmd-apex-jorje/pom.xml | 2 +- pmd-apex/pom.xml | 2 +- pmd-core/pom.xml | 2 +- pmd-cpp/pom.xml | 2 +- pmd-cs/pom.xml | 2 +- pmd-dart/pom.xml | 2 +- pmd-dist/pom.xml | 2 +- pmd-doc/pom.xml | 2 +- pmd-fortran/pom.xml | 2 +- pmd-gherkin/pom.xml | 2 +- pmd-go/pom.xml | 2 +- pmd-groovy/pom.xml | 2 +- pmd-html/pom.xml | 2 +- pmd-java/pom.xml | 2 +- pmd-java8/pom.xml | 2 +- pmd-javascript/pom.xml | 2 +- pmd-jsp/pom.xml | 2 +- pmd-kotlin/pom.xml | 2 +- pmd-lang-test/pom.xml | 2 +- pmd-lua/pom.xml | 2 +- pmd-matlab/pom.xml | 2 +- pmd-modelica/pom.xml | 2 +- pmd-objectivec/pom.xml | 2 +- pmd-perl/pom.xml | 2 +- pmd-php/pom.xml | 2 +- pmd-plsql/pom.xml | 2 +- pmd-python/pom.xml | 2 +- pmd-ruby/pom.xml | 2 +- pmd-scala-modules/pmd-scala-common/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.12/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.13/pom.xml | 2 +- pmd-scala/pom.xml | 2 +- pmd-swift/pom.xml | 2 +- pmd-test-schema/pom.xml | 2 +- pmd-test/pom.xml | 2 +- pmd-visualforce/pom.xml | 2 +- pmd-vm/pom.xml | 2 +- pmd-xml/pom.xml | 2 +- pom.xml | 6 +++--- 39 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index c605e61a9d..9d82fcc6e2 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-apex/pom.xml b/pmd-apex/pom.xml index a661b0e3c8..64d14d5a86 100644 --- a/pmd-apex/pom.xml +++ b/pmd-apex/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index 4ef2c6c1c1..f43367694a 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-cpp/pom.xml b/pmd-cpp/pom.xml index 839b79e061..6f3d3afcee 100644 --- a/pmd-cpp/pom.xml +++ b/pmd-cpp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-cs/pom.xml b/pmd-cs/pom.xml index b6169ce0f5..345f57f90b 100644 --- a/pmd-cs/pom.xml +++ b/pmd-cs/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-dart/pom.xml b/pmd-dart/pom.xml index 4b29f14fbc..8989f246ce 100644 --- a/pmd-dart/pom.xml +++ b/pmd-dart/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-dist/pom.xml b/pmd-dist/pom.xml index 0624ab525a..c5912ee704 100644 --- a/pmd-dist/pom.xml +++ b/pmd-dist/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-doc/pom.xml b/pmd-doc/pom.xml index 1dd433f596..d53987ee65 100644 --- a/pmd-doc/pom.xml +++ b/pmd-doc/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-fortran/pom.xml b/pmd-fortran/pom.xml index c26c301c30..c092b17d49 100644 --- a/pmd-fortran/pom.xml +++ b/pmd-fortran/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-gherkin/pom.xml b/pmd-gherkin/pom.xml index 20eda17f8a..047f06705d 100644 --- a/pmd-gherkin/pom.xml +++ b/pmd-gherkin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-go/pom.xml b/pmd-go/pom.xml index febadeb629..8fff13187f 100644 --- a/pmd-go/pom.xml +++ b/pmd-go/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-groovy/pom.xml b/pmd-groovy/pom.xml index 5fea6def0c..4f04a88d66 100644 --- a/pmd-groovy/pom.xml +++ b/pmd-groovy/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-html/pom.xml b/pmd-html/pom.xml index 381b86a9a0..70c45debaa 100644 --- a/pmd-html/pom.xml +++ b/pmd-html/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-java/pom.xml b/pmd-java/pom.xml index ac5a921e0e..1f0ca4dae9 100644 --- a/pmd-java/pom.xml +++ b/pmd-java/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-java8/pom.xml b/pmd-java8/pom.xml index 47a06c8528..6a4c97c7c6 100644 --- a/pmd-java8/pom.xml +++ b/pmd-java8/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-javascript/pom.xml b/pmd-javascript/pom.xml index 844634afde..0c58e2aef2 100644 --- a/pmd-javascript/pom.xml +++ b/pmd-javascript/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-jsp/pom.xml b/pmd-jsp/pom.xml index c027744cf0..14f1f6d76b 100644 --- a/pmd-jsp/pom.xml +++ b/pmd-jsp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-kotlin/pom.xml b/pmd-kotlin/pom.xml index 4e58d15787..125ac5cdc0 100644 --- a/pmd-kotlin/pom.xml +++ b/pmd-kotlin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-lang-test/pom.xml b/pmd-lang-test/pom.xml index 1d499c152a..f53f195b22 100644 --- a/pmd-lang-test/pom.xml +++ b/pmd-lang-test/pom.xml @@ -12,7 +12,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-lua/pom.xml b/pmd-lua/pom.xml index b5505b035e..f342207fd0 100644 --- a/pmd-lua/pom.xml +++ b/pmd-lua/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-matlab/pom.xml b/pmd-matlab/pom.xml index 30e061f46b..be40507861 100644 --- a/pmd-matlab/pom.xml +++ b/pmd-matlab/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-modelica/pom.xml b/pmd-modelica/pom.xml index 48bfa8a679..72b7a50b8f 100644 --- a/pmd-modelica/pom.xml +++ b/pmd-modelica/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-objectivec/pom.xml b/pmd-objectivec/pom.xml index 93a391f0f6..4aee2cf7dc 100644 --- a/pmd-objectivec/pom.xml +++ b/pmd-objectivec/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-perl/pom.xml b/pmd-perl/pom.xml index cf0886bd93..9a44d3d7d7 100644 --- a/pmd-perl/pom.xml +++ b/pmd-perl/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-php/pom.xml b/pmd-php/pom.xml index 6f772c36bb..af01ee2f6f 100644 --- a/pmd-php/pom.xml +++ b/pmd-php/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-plsql/pom.xml b/pmd-plsql/pom.xml index d3bbc1bc3b..b3cf518cce 100644 --- a/pmd-plsql/pom.xml +++ b/pmd-plsql/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-python/pom.xml b/pmd-python/pom.xml index ec6c5e4c30..9c30438d09 100644 --- a/pmd-python/pom.xml +++ b/pmd-python/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-ruby/pom.xml b/pmd-ruby/pom.xml index 532e67da47..62b5d6b15b 100644 --- a/pmd-ruby/pom.xml +++ b/pmd-ruby/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-scala-modules/pmd-scala-common/pom.xml b/pmd-scala-modules/pmd-scala-common/pom.xml index b303122d0c..14ac1f8367 100644 --- a/pmd-scala-modules/pmd-scala-common/pom.xml +++ b/pmd-scala-modules/pmd-scala-common/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../../pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.12/pom.xml b/pmd-scala-modules/pmd-scala_2.12/pom.xml index eadca460a0..61ac0b7d84 100644 --- a/pmd-scala-modules/pmd-scala_2.12/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.12/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.48.0 + 6.49.0-SNAPSHOT ../pmd-scala-common/pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.13/pom.xml b/pmd-scala-modules/pmd-scala_2.13/pom.xml index 1468d708b5..6bf91dcd09 100644 --- a/pmd-scala-modules/pmd-scala_2.13/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.13/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.48.0 + 6.49.0-SNAPSHOT ../pmd-scala-common/pom.xml diff --git a/pmd-scala/pom.xml b/pmd-scala/pom.xml index bdd29afbab..c12939498b 100644 --- a/pmd-scala/pom.xml +++ b/pmd-scala/pom.xml @@ -9,7 +9,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-swift/pom.xml b/pmd-swift/pom.xml index f19edbaa50..43699a38b0 100644 --- a/pmd-swift/pom.xml +++ b/pmd-swift/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-test-schema/pom.xml b/pmd-test-schema/pom.xml index 2f15a89727..5bb6d34c03 100644 --- a/pmd-test-schema/pom.xml +++ b/pmd-test-schema/pom.xml @@ -11,7 +11,7 @@ pmd net.sourceforge.pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-test/pom.xml b/pmd-test/pom.xml index 594a0619fa..50e48f3a30 100644 --- a/pmd-test/pom.xml +++ b/pmd-test/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-visualforce/pom.xml b/pmd-visualforce/pom.xml index de6d255a9a..f66ecc00cb 100644 --- a/pmd-visualforce/pom.xml +++ b/pmd-visualforce/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-vm/pom.xml b/pmd-vm/pom.xml index 3e1e175787..597785ae98 100644 --- a/pmd-vm/pom.xml +++ b/pmd-vm/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pmd-xml/pom.xml b/pmd-xml/pom.xml index 58804856eb..3b5e97f73e 100644 --- a/pmd-xml/pom.xml +++ b/pmd-xml/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 91a7b0c728..aef9c8a492 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 net.sourceforge.pmd pmd - 6.48.0 + 6.49.0-SNAPSHOT pom PMD @@ -55,7 +55,7 @@ scm:git:git://github.com/pmd/pmd.git scm:git:ssh://git@github.com/pmd/pmd.git https://github.com/pmd/pmd - pmd_releases/6.48.0 + HEAD @@ -76,7 +76,7 @@ - 2022-07-30T09:23:49Z + 2022-07-30T09:35:57Z 7 From 77e3bab4a62f33834c76970a8d177b48aab9125b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 11:38:19 +0200 Subject: [PATCH 082/136] Prepare next development version [skip ci] --- docs/_config.yml | 6 +- docs/pages/release_notes.md | 119 ------------------------- docs/pages/release_notes_old.md | 149 ++++++++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 122 deletions(-) diff --git a/docs/_config.yml b/docs/_config.yml index 5ccd6a03b2..75b1ae16eb 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,9 +1,9 @@ repository: pmd/pmd pmd: - version: 6.48.0 - previous_version: 6.47.0 - date: 30-July-2022 + version: 6.49.0-SNAPSHOT + previous_version: 6.48.0 + date: 31-August-2022 release_type: minor # release types: major, minor, bugfix diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index eeabe3a7dc..b8f8783555 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,130 +14,11 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy -#### Java 19 Support - -This release of PMD brings support for Java 19. There are no new standard language features. - -PMD supports [JEP 427: Pattern Matching for switch (Third Preview)](https://openjdk.org/jeps/427) and -[JEP 405: Record Patterns (Preview)](https://openjdk.org/jeps/405) as preview language features. - -In order to analyze a project with PMD that uses these language features, -you'll need to enable it via the environment variable `PMD_JAVA_OPTS` and select the new language -version `19-preview`: - - export PMD_JAVA_OPTS=--enable-preview - ./run.sh pmd -language java -version 19-preview ... - -Note: Support for Java 17 preview language features have been removed. The version "17-preview" is no longer available. - -#### Gherkin support -Thanks to the contribution from [Anne Brouwers](https://github.com/ASBrouwers) PMD now has CPD support -for the [Gherkin](https://cucumber.io/docs/gherkin/) language. It is used to defined test cases for the -[Cucumber](https://cucumber.io/) testing tool for behavior-driven development. - -Being based on a proper Antlr grammar, CPD can: - -* ignore comments -* honor [comment-based suppressions](pmd_userdocs_cpd.html#suppression) - ### Fixed Issues -* apex - * [#4056](https://github.com/pmd/pmd/pull/4056): \[apex] ApexSOQLInjection: Add support count query -* core - * [#3796](https://github.com/pmd/pmd/issues/3796): \[core] CPD should also provide a `--debug` flag - * [#4021](https://github.com/pmd/pmd/pull/4021): \[core] CPD: Add total number of tokens to XML reports - * [#4031](https://github.com/pmd/pmd/issues/4031): \[core] If report is written to stdout, stdout should not be closed - * [#4051](https://github.com/pmd/pmd/issues/4051): \[doc] Additional rulesets are not listed in documentation - * [#4053](https://github.com/pmd/pmd/pull/4053): \[core] Allow building PMD under Java 18+ -* java - * [#4015](https://github.com/pmd/pmd/issues/4015): \[java] Support JDK 19 -* java-bestpractices - * [#3455](https://github.com/pmd/pmd/issues/3455): \[java] WhileLoopWithLiteralBoolean - false negative with complex expressions -* java-design - * [#3729](https://github.com/pmd/pmd/issues/3729): \[java] TooManyMethods ignores "real" methods which are named like getters or setters - * [#3949](https://github.com/pmd/pmd/issues/3949): \[java] FinalFieldCouldBeStatic - false negative with unnecessary parenthesis -* java-performance - * [#3625](https://github.com/pmd/pmd/issues/3625): \[java] AddEmptyString - false negative with empty var -* lua - * [#4061](https://github.com/pmd/pmd/pull/4061): \[lua] Fix several related Lua parsing issues found when using CPD -* test - * [#3302](https://github.com/pmd/pmd/pull/3302): \[test] Improve xml test schema - * [#3758](https://github.com/pmd/pmd/issues/3758): \[test] Move pmd-test to java 8 - * [#3976](https://github.com/pmd/pmd/pull/3976): \[test] Extract xml schema module ### API Changes -#### CPD CLI - -* CPD has a new CLI option `--debug`. This option has the same behavior as in PMD. It enables more verbose - logging output. - -#### Rule Test Framework - -* The module "pmd-test", which contains support classes to write rule tests, now **requires Java 8**. If you depend on - this module for testing your own custom rules, you'll need to make sure to use at least Java 8. -* The new module "pmd-test-schema" contains now the XSD schema and the code to parse the rule test XML files. The - schema has been extracted in order to easily share it with other tools like the Rule Designer or IDE plugins. -* Test schema changes: - * The attribute `isRegressionTest` of `test-code` is deprecated. The new - attribute `disabled` should be used instead for defining whether a rule test should be skipped or not. - * The attributes `reinitializeRule` and `useAuxClasspath` of `test-code` are deprecated and assumed true. - They will not be replaced. - * The new attribute `focused` of `test-code` allows disabling all tests except the focused one temporarily. -* More information about the rule test framework can be found in the documentation: - [Testing your rules](pmd_userdocs_extending_testing.html) - -#### Deprecated API - -* The experimental Java AST class {% jdoc java::lang.java.ast.ASTGuardedPattern %} has been deprecated and - will be removed. It was introduced for Java 17 and Java 18 Preview as part of pattern matching for switch, - but it is no longer supported with Java 19 Preview. -* The interface {% jdoc core::cpd.renderer.CPDRenderer %} is deprecated. For custom CPD renderers - the new interface {% jdoc core::cpd.renderer.CPDReportRenderer %} should be used. -* The class {% jdoc test::testframework.TestDescriptor %} is deprecated, replaced with {% jdoc test-schema::test.schema.RuleTestDescriptor %}. -* Many methods of {% jdoc test::testframework.RuleTst %} have been deprecated as internal API. - -#### Experimental APIs - -* To support the Java preview language features "Pattern Matching for Switch" and "Record Patterns", the following - AST nodes have been introduced as experimental: - * {% jdoc java::lang.java.ast.ASTSwitchGuard %} - * {% jdoc java::lang.java.ast.ASTRecordPattern %} - * {% jdoc java::lang.java.ast.ASTComponentPatternList %} - -#### Internal API - -Those APIs are not intended to be used by clients, and will be hidden or removed with PMD 7.0.0. -You can identify them with the `@InternalApi` annotation. You'll also get a deprecation warning. - -* {%jdoc !!core::cpd.CPDConfiguration#setRenderer(net.sourceforge.pmd.cpd.Renderer) %} -* {%jdoc !!core::cpd.CPDConfiguration#setCPDRenderer(net.sourceforge.pmd.cpd.renderer.CPDRenderer) %} -* {%jdoc !!core::cpd.CPDConfiguration#getRenderer() %} -* {%jdoc !!core::cpd.CPDConfiguration#getCPDRenderer() %} -* {%jdoc !!core::cpd.CPDConfiguration#getRendererFromString(java.lang.String,java.lang.String) %} -* {%jdoc !!core::cpd.CPDConfiguration#getCPDRendererFromString(java.lang.String,java.lang.String) %} -* {%jdoc core::cpd.renderer.CPDRendererAdapter %} - -### Financial Contributions - -Many thanks to our sponsors: - -* [Matt Hargett](https://github.com/matthargett) (@matthargett) - ### External Contributions -* [#3984](https://github.com/pmd/pmd/pull/3984): \[java] Fix AddEmptyString false-negative issue - [@LiGaOg](https://github.com/LiGaOg) -* [#3988](https://github.com/pmd/pmd/pull/3988): \[java] Modify WhileLoopWithLiteralBoolean to meet the missing case #3455 - [@VoidxHoshi](https://github.com/VoidxHoshi) -* [#3992](https://github.com/pmd/pmd/pull/3992): \[java] FinalFieldCouldBeStatic - fix false negative with unnecessary parenthesis - [@dalizi007](https://github.com/dalizi007) -* [#3994](https://github.com/pmd/pmd/pull/3994): \[java] TooManyMethods - improve getter/setter detection (#3729) - [@341816041](https://github.com/341816041) -* [#4017](https://github.com/pmd/pmd/pull/4017): Add Gherkin support to CPD - [@ASBrouwers](https://github.com/ASBrouwers) -* [#4021](https://github.com/pmd/pmd/pull/4021): \[core] CPD: Add total number of tokens to XML reports - [@maikelsteneker](https://github.com/maikelsteneker) -* [#4056](https://github.com/pmd/pmd/pull/4056): \[apex] ApexSOQLInjection: Add support count query - [@gwilymatgearset](https://github.com/gwilymatgearset) -* [#4061](https://github.com/pmd/pmd/pull/4061): \[lua] Fix several related Lua parsing issues found when using CPD - [@matthargett](https://github.com/matthargett) - -### Stats -* 102 commits -* 26 closed tickets & PRs -* Days since last release: 35 {% endtocmaker %} diff --git a/docs/pages/release_notes_old.md b/docs/pages/release_notes_old.md index e5b53bc05d..cd56c15d74 100644 --- a/docs/pages/release_notes_old.md +++ b/docs/pages/release_notes_old.md @@ -5,6 +5,155 @@ permalink: pmd_release_notes_old.html Previous versions of PMD can be downloaded here: https://github.com/pmd/pmd/releases +## 30-July-2022 - 6.48.0 + +The PMD team is pleased to announce PMD 6.48.0. + +This is a minor release. + +### Table Of Contents + +* [New and noteworthy](#new-and-noteworthy) + * [Java 19 Support](#java-19-support) + * [Gherkin support](#gherkin-support) +* [Fixed Issues](#fixed-issues) +* [API Changes](#api-changes) + * [CPD CLI](#cpd-cli) + * [Rule Test Framework](#rule-test-framework) + * [Deprecated API](#deprecated-api) + * [Experimental APIs](#experimental-apis) + * [Internal API](#internal-api) +* [Financial Contributions](#financial-contributions) +* [External Contributions](#external-contributions) +* [Stats](#stats) + +### New and noteworthy + +#### Java 19 Support + +This release of PMD brings support for Java 19. There are no new standard language features. + +PMD supports [JEP 427: Pattern Matching for switch (Third Preview)](https://openjdk.org/jeps/427) and +[JEP 405: Record Patterns (Preview)](https://openjdk.org/jeps/405) as preview language features. + +In order to analyze a project with PMD that uses these language features, +you'll need to enable it via the environment variable `PMD_JAVA_OPTS` and select the new language +version `19-preview`: + + export PMD_JAVA_OPTS=--enable-preview + ./run.sh pmd -language java -version 19-preview ... + +Note: Support for Java 17 preview language features have been removed. The version "17-preview" is no longer available. + +#### Gherkin support +Thanks to the contribution from [Anne Brouwers](https://github.com/ASBrouwers) PMD now has CPD support +for the [Gherkin](https://cucumber.io/docs/gherkin/) language. It is used to defined test cases for the +[Cucumber](https://cucumber.io/) testing tool for behavior-driven development. + +Being based on a proper Antlr grammar, CPD can: + +* ignore comments +* honor [comment-based suppressions](pmd_userdocs_cpd.html#suppression) + +### Fixed Issues +* apex + * [#4056](https://github.com/pmd/pmd/pull/4056): \[apex] ApexSOQLInjection: Add support count query +* core + * [#3796](https://github.com/pmd/pmd/issues/3796): \[core] CPD should also provide a `--debug` flag + * [#4021](https://github.com/pmd/pmd/pull/4021): \[core] CPD: Add total number of tokens to XML reports + * [#4031](https://github.com/pmd/pmd/issues/4031): \[core] If report is written to stdout, stdout should not be closed + * [#4051](https://github.com/pmd/pmd/issues/4051): \[doc] Additional rulesets are not listed in documentation + * [#4053](https://github.com/pmd/pmd/pull/4053): \[core] Allow building PMD under Java 18+ +* java + * [#4015](https://github.com/pmd/pmd/issues/4015): \[java] Support JDK 19 +* java-bestpractices + * [#3455](https://github.com/pmd/pmd/issues/3455): \[java] WhileLoopWithLiteralBoolean - false negative with complex expressions +* java-design + * [#3729](https://github.com/pmd/pmd/issues/3729): \[java] TooManyMethods ignores "real" methods which are named like getters or setters + * [#3949](https://github.com/pmd/pmd/issues/3949): \[java] FinalFieldCouldBeStatic - false negative with unnecessary parenthesis +* java-performance + * [#3625](https://github.com/pmd/pmd/issues/3625): \[java] AddEmptyString - false negative with empty var +* lua + * [#4061](https://github.com/pmd/pmd/pull/4061): \[lua] Fix several related Lua parsing issues found when using CPD +* test + * [#3302](https://github.com/pmd/pmd/pull/3302): \[test] Improve xml test schema + * [#3758](https://github.com/pmd/pmd/issues/3758): \[test] Move pmd-test to java 8 + * [#3976](https://github.com/pmd/pmd/pull/3976): \[test] Extract xml schema module + +### API Changes + +#### CPD CLI + +* CPD has a new CLI option `--debug`. This option has the same behavior as in PMD. It enables more verbose + logging output. + +#### Rule Test Framework + +* The module "pmd-test", which contains support classes to write rule tests, now **requires Java 8**. If you depend on + this module for testing your own custom rules, you'll need to make sure to use at least Java 8. +* The new module "pmd-test-schema" contains now the XSD schema and the code to parse the rule test XML files. The + schema has been extracted in order to easily share it with other tools like the Rule Designer or IDE plugins. +* Test schema changes: + * The attribute `isRegressionTest` of `test-code` is deprecated. The new + attribute `disabled` should be used instead for defining whether a rule test should be skipped or not. + * The attributes `reinitializeRule` and `useAuxClasspath` of `test-code` are deprecated and assumed true. + They will not be replaced. + * The new attribute `focused` of `test-code` allows disabling all tests except the focused one temporarily. +* More information about the rule test framework can be found in the documentation: + [Testing your rules](pmd_userdocs_extending_testing.html) + +#### Deprecated API + +* The experimental Java AST class ASTGuardedPattern has been deprecated and + will be removed. It was introduced for Java 17 and Java 18 Preview as part of pattern matching for switch, + but it is no longer supported with Java 19 Preview. +* The interface CPDRenderer is deprecated. For custom CPD renderers + the new interface CPDReportRenderer should be used. +* The class TestDescriptor is deprecated, replaced with RuleTestDescriptor. +* Many methods of RuleTst have been deprecated as internal API. + +#### Experimental APIs + +* To support the Java preview language features "Pattern Matching for Switch" and "Record Patterns", the following + AST nodes have been introduced as experimental: + * ASTSwitchGuard + * ASTRecordPattern + * ASTComponentPatternList + +#### Internal API + +Those APIs are not intended to be used by clients, and will be hidden or removed with PMD 7.0.0. +You can identify them with the `@InternalApi` annotation. You'll also get a deprecation warning. + +* CPDConfiguration#setRenderer +* CPDConfiguration#setCPDRenderer +* CPDConfiguration#getRenderer +* CPDConfiguration#getCPDRenderer +* CPDConfiguration#getRendererFromString +* CPDConfiguration#getCPDRendererFromString +* CPDRendererAdapter + +### Financial Contributions + +Many thanks to our sponsors: + +* [Matt Hargett](https://github.com/matthargett) (@matthargett) + +### External Contributions +* [#3984](https://github.com/pmd/pmd/pull/3984): \[java] Fix AddEmptyString false-negative issue - [@LiGaOg](https://github.com/LiGaOg) +* [#3988](https://github.com/pmd/pmd/pull/3988): \[java] Modify WhileLoopWithLiteralBoolean to meet the missing case #3455 - [@VoidxHoshi](https://github.com/VoidxHoshi) +* [#3992](https://github.com/pmd/pmd/pull/3992): \[java] FinalFieldCouldBeStatic - fix false negative with unnecessary parenthesis - [@dalizi007](https://github.com/dalizi007) +* [#3994](https://github.com/pmd/pmd/pull/3994): \[java] TooManyMethods - improve getter/setter detection (#3729) - [@341816041](https://github.com/341816041) +* [#4017](https://github.com/pmd/pmd/pull/4017): Add Gherkin support to CPD - [@ASBrouwers](https://github.com/ASBrouwers) +* [#4021](https://github.com/pmd/pmd/pull/4021): \[core] CPD: Add total number of tokens to XML reports - [@maikelsteneker](https://github.com/maikelsteneker) +* [#4056](https://github.com/pmd/pmd/pull/4056): \[apex] ApexSOQLInjection: Add support count query - [@gwilymatgearset](https://github.com/gwilymatgearset) +* [#4061](https://github.com/pmd/pmd/pull/4061): \[lua] Fix several related Lua parsing issues found when using CPD - [@matthargett](https://github.com/matthargett) + +### Stats +* 102 commits +* 26 closed tickets & PRs +* Days since last release: 35 + ## 25-June-2022 - 6.47.0 The PMD team is pleased to announce PMD 6.47.0. From 533428db9b1cecbfed998890f5407234590d0500 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 30 Jul 2022 12:23:19 +0200 Subject: [PATCH 083/136] Bump pmd from 6.47.0 to 6.48.0 --- pom.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pom.xml b/pom.xml index aef9c8a492..1f7aa9e9b7 100644 --- a/pom.xml +++ b/pom.xml @@ -407,22 +407,22 @@ net.sourceforge.pmd pmd-core - 6.47.0 + 6.48.0 net.sourceforge.pmd pmd-java - 6.47.0 + 6.48.0 net.sourceforge.pmd pmd-jsp - 6.47.0 + 6.48.0 net.sourceforge.pmd pmd-javascript - 6.47.0 + 6.48.0 From dedb263fe781ad3ccdd5d91af705079feb9e6fd0 Mon Sep 17 00:00:00 2001 From: Simon Abykov Date: Fri, 5 Aug 2022 11:34:01 +0100 Subject: [PATCH 084/136] [java] UnnecessaryImport false positive for on-demand imports of nested classes Fixes #4082 --- .../rule/codestyle/UnnecessaryImportRule.java | 28 +++++++++++++++++++ .../unnecessaryimport/package2/C.java | 4 +++ .../rule/codestyle/xml/UnnecessaryImport.xml | 28 +++++++++++++++++++ 3 files changed, 60 insertions(+) diff --git a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/UnnecessaryImportRule.java b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/UnnecessaryImportRule.java index 3359462489..28ff548dbc 100644 --- a/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/UnnecessaryImportRule.java +++ b/pmd-java/src/main/java/net/sourceforge/pmd/lang/java/rule/codestyle/UnnecessaryImportRule.java @@ -210,6 +210,20 @@ public class UnnecessaryImportRule extends AbstractJavaRule { } } + // check on-demand imports for inner classes + it = imports.iterator(); + while (it.hasNext()) { + ImportWrapper i = it.next(); + if (!i.isStaticOnDemand() && i.isOnDemand()) { + String possibleClassName = i.getFullName() + "$" + candName; + Class possibleClazz = referenceNode.getRoot().getClassTypeResolver() + .loadClassOrNull(possibleClassName); + if (possibleClazz != null) { + it.remove(); + } + } + } + // check static on-demand imports it = imports.iterator(); while (it.hasNext()) { @@ -220,6 +234,20 @@ public class UnnecessaryImportRule extends AbstractJavaRule { } } + // check static on-demand imports for static inner classes + it = imports.iterator(); + while (it.hasNext()) { + ImportWrapper i = it.next(); + if (i.isStaticOnDemand()) { + String possibleClassName = i.getFullName() + "$" + candName; + Class possibleClazz = referenceNode.getRoot().getClassTypeResolver() + .loadClassOrNull(possibleClassName); + if (possibleClazz != null) { + it.remove(); + } + } + } + if (referenceNode instanceof TypeNode && ((TypeNode) referenceNode).getType() != null) { Class c = ((TypeNode) referenceNode).getType(); if (c.getPackage() != null) { diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java index 19710af874..43be196748 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java @@ -7,5 +7,9 @@ package net.sourceforge.pmd.lang.java.rule.codestyle.unnecessaryimport.package2; public class C { private C() { } + public class IC {} + + public static class ISC {} + public static final String V = ""; } diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml index c1c8fb2e30..d18dbdd54e 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml @@ -916,6 +916,34 @@ public class U { ]]> + + [java] UnnecessaryImport false positive for on-demand imports of non-static nested classes + 0 + + + + + [java] UnnecessaryImport false positive for static on-demand imports of static nested classes + 0 + + + Necessary imports for @snippet tags introduced with JEP 413 in Java 18 0 From b63368fe8d962d3dda08e92a7628399c06f80cad Mon Sep 17 00:00:00 2001 From: Simon Abykov Date: Fri, 5 Aug 2022 13:43:51 +0100 Subject: [PATCH 085/136] Fixed checkstyle errors --- .../java/rule/codestyle/unnecessaryimport/package2/C.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java index 43be196748..92104cbbb1 100644 --- a/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java +++ b/pmd-java/src/test/java/net/sourceforge/pmd/lang/java/rule/codestyle/unnecessaryimport/package2/C.java @@ -7,9 +7,9 @@ package net.sourceforge.pmd.lang.java.rule.codestyle.unnecessaryimport.package2; public class C { private C() { } - public class IC {} + public class IC { } - public static class ISC {} + public static class ISC { } public static final String V = ""; } From 41c924545c67df1791d1ad4d90f90b453d6cb8fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 10 Aug 2022 00:12:57 +0200 Subject: [PATCH 086/136] Remove deprecated attribute usages in test files --- .../bestpractices/xml/AvoidReassigningLoopVariables.xml | 2 +- .../lang/java/rule/bestpractices/xml/PreserveStackTrace.xml | 6 +++--- .../lang/java/rule/bestpractices/xml/UnusedAssignment.xml | 2 +- .../rule/codestyle/xml/UnnecessaryFullyQualifiedName.xml | 2 +- .../pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml | 2 +- .../pmd/lang/java/rule/codestyle/xml/UnnecessaryReturn.xml | 2 +- .../java/rule/errorprone/xml/BeanMembersShouldSerialize.xml | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AvoidReassigningLoopVariables.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AvoidReassigningLoopVariables.xml index d002c076ef..8180b410be 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AvoidReassigningLoopVariables.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/AvoidReassigningLoopVariables.xml @@ -674,7 +674,7 @@ public class Foo { - + violation: various conditional reassignments of 'for' loop variable, skip allowed skip 4 diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/PreserveStackTrace.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/PreserveStackTrace.xml index a20b8531d1..5292c61d64 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/PreserveStackTrace.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/PreserveStackTrace.xml @@ -738,7 +738,7 @@ public class SomeOtherUserDefinedException extends Exception { } ]]> - + FP with reassigned exception 1 @@ -755,7 +755,7 @@ public class SomeOtherUserDefinedException extends Exception { } ]]> - + FP with reassigned exception (branch) 1 @@ -1080,7 +1080,7 @@ public class SomeOtherUserDefinedException extends Exception { ]]> - + Branch in dataflow should be merged 2 diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedAssignment.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedAssignment.xml index 7f4341aa5e..d9aad8af15 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedAssignment.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/bestpractices/xml/UnusedAssignment.xml @@ -3289,7 +3289,7 @@ public class UnusedAssignmentNative { ]]> - + Explicit this ctor call 1 diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryFullyQualifiedName.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryFullyQualifiedName.xml index 85f6c7871c..e5a1609bbb 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryFullyQualifiedName.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryFullyQualifiedName.xml @@ -883,7 +883,7 @@ class Assert { } ]]> - + FN with static method imported through subtype (same file) diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml index 1d5e88a07e..6e774c7040 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryImport.xml @@ -158,7 +158,7 @@ public class Foo { ]]> - + Used static import 0 diff --git a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryReturn.xml b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryReturn.xml index 09a62b6d1b..e220540880 100644 --- a/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryReturn.xml +++ b/pmd-java/src/test/resources/net/sourceforge/pmd/lang/java/rule/codestyle/xml/UnnecessaryReturn.xml @@ -365,7 +365,7 @@ public class Foo { ]]> - + Ignore local class statements 1 - + #780 [java] BeanMembersShouldSerializeRule does not recognize lombok accessors - 3 0 Date: Wed, 10 Aug 2022 00:18:21 +0200 Subject: [PATCH 087/136] test-schema: fix wrong warning message --- .../net/sourceforge/pmd/test/schema/BaseTestParserImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java index 192b11ba4a..761fa508bf 100644 --- a/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java +++ b/pmd-test-schema/src/main/java/net/sourceforge/pmd/test/schema/BaseTestParserImpl.java @@ -96,7 +96,7 @@ class BaseTestParserImpl { parseBoolAttribute(testCode, "useAuxClasspath", true, err, "Attribute 'useAuxClasspath' is deprecated and ignored, assumed true"); boolean disabled = parseBoolAttribute(testCode, "disabled", false, err, null) - | !parseBoolAttribute(testCode, "regressionTest", true, err, "Attribute ''regressionTest'' is deprecated, use ''ignored'' with inverted value"); + | !parseBoolAttribute(testCode, "regressionTest", true, err, "Attribute ''regressionTest'' is deprecated, use ''disabled'' with inverted value"); descriptor.setDisabled(disabled); From 2977bd8764ef01152e9f74b2786ee43b937b5379 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Fournier?= Date: Wed, 10 Aug 2022 17:26:20 +0200 Subject: [PATCH 088/136] test-schema: fix tests --- .../net/sourceforge/pmd/test/schema/TestSchemaParserTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java b/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java index 1b38d243f6..286b61cd8e 100644 --- a/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java +++ b/pmd-test-schema/src/test/java/net/sourceforge/pmd/test/schema/TestSchemaParserTest.java @@ -82,7 +82,7 @@ public class TestSchemaParserTest { assertEquals(1, parsed.getTests().size()); MatcherAssert.assertThat(errStreamCaptor.getLog(), containsString(" 6| \n" - + " ^^^^^^^^^^^^^^ Attribute 'regressionTest' is deprecated, use 'ignored' with inverted value\n")); + + " ^^^^^^^^^^^^^^ Attribute 'regressionTest' is deprecated, use 'disabled' with inverted value\n")); } @Test From ea455fa356ff0805ff86527762c990869ad28294 Mon Sep 17 00:00:00 2001 From: Edward Klimoshenko Date: Tue, 19 Jul 2022 04:30:05 +0000 Subject: [PATCH 089/136] Remove Jorje leaks outside `ast` package Operators - Create operator enums to wrap internal Jorje operators. - Create getters returning new operator enums in operator-expression nodes. Deprecate old getters. - Refactor metrics classes to use new operator enums. Tests - Remove references to internal Jorje nodes. --- .../apex/ast/ASTAssignmentExpression.java | 5 ++ .../lang/apex/ast/ASTBinaryExpression.java | 5 ++ .../lang/apex/ast/ASTBooleanExpression.java | 5 +- .../lang/apex/ast/ASTPostfixExpression.java | 6 +- .../lang/apex/ast/ASTPrefixExpression.java | 6 +- .../pmd/lang/apex/ast/ApexRootNode.java | 4 +- .../pmd/lang/apex/ast/AssignmentOperator.java | 67 +++++++++++++++++++ .../pmd/lang/apex/ast/BinaryOperator.java | 64 ++++++++++++++++++ .../pmd/lang/apex/ast/BooleanOperator.java | 67 +++++++++++++++++++ .../pmd/lang/apex/ast/PostfixOperator.java | 40 +++++++++++ .../pmd/lang/apex/ast/PrefixOperator.java | 52 ++++++++++++++ .../lang/apex/metrics/impl/CycloMetric.java | 7 +- .../visitors/CognitiveComplexityVisitor.java | 17 +++-- .../pmd/lang/apex/ast/ASTFieldTest.java | 8 +-- .../pmd/lang/apex/ast/ASTMethodTest.java | 4 +- .../ASTNewKeyValueObjectExpressionTest.java | 4 +- .../lang/apex/ast/ASTSoqlExpressionTest.java | 4 +- .../lang/apex/ast/ASTSwitchStatementTest.java | 4 +- .../ASTTryCatchFinallyBlockStatementTest.java | 8 +-- .../pmd/lang/apex/ast/ASTUserClassTest.java | 6 +- .../pmd/lang/apex/ast/ASTUserEnumTest.java | 4 +- .../lang/apex/ast/ASTUserInterfaceTest.java | 6 +- .../pmd/lang/apex/ast/ApexParserTest.java | 22 +++--- .../pmd/lang/apex/ast/ApexParserTestBase.java | 6 +- .../lang/apex/ast/ApexQualifiedNameTest.java | 17 ++--- .../apex/metrics/ApexProjectMirrorTest.java | 6 +- .../multifile/ApexMultifileVisitorTest.java | 4 +- .../lang/apex/rule/AbstractApexRuleTest.java | 4 +- 28 files changed, 366 insertions(+), 86 deletions(-) create mode 100644 pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/AssignmentOperator.java create mode 100644 pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BinaryOperator.java create mode 100644 pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BooleanOperator.java create mode 100644 pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PostfixOperator.java create mode 100644 pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PrefixOperator.java diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java index 64db7c1687..73b3b10740 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java @@ -22,7 +22,12 @@ public class ASTAssignmentExpression extends AbstractApexNode { return visitor.visit(this, data); } + @Deprecated public BinaryOp getOperator() { return node.getOp(); } + + public BinaryOperator getOp() { + return BinaryOperator.valueOf(node.getOp()); + } } diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java index c6ab8b415e..f258a389c8 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java @@ -24,9 +24,12 @@ public class ASTBooleanExpression extends AbstractApexNode { return visitor.visit(this, data); } - + @Deprecated public BooleanOp getOperator() { return this.node.getOp(); } + public BooleanOperator getOp() { + return BooleanOperator.valueOf(this.node.getOp()); + } } diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java index 5d3c288fbc..8ef0b8770b 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java @@ -24,8 +24,12 @@ public class ASTPostfixExpression extends AbstractApexNode { return visitor.visit(this, data); } - + @Deprecated public PostfixOp getOperator() { return node.getOp(); } + + public PostfixOperator getOp() { + return PostfixOperator.valueOf(node.getOp()); + } } diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java index c79fa8214f..bc8c4fdf29 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java @@ -22,8 +22,12 @@ public class ASTPrefixExpression extends AbstractApexNode { return visitor.visit(this, data); } - + @Deprecated public PrefixOp getOperator() { return node.getOp(); } + + public PrefixOperator getOp() { + return PrefixOperator.valueOf(node.getOp()); + } } diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexRootNode.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexRootNode.java index a14c241b44..916e7cd3e5 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexRootNode.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexRootNode.java @@ -8,12 +8,12 @@ import net.sourceforge.pmd.annotation.InternalApi; import net.sourceforge.pmd.lang.ast.RootNode; import net.sourceforge.pmd.lang.ast.SourceCodePositioner; -import apex.jorje.semantic.ast.AstNode; +import apex.jorje.semantic.ast.compilation.Compilation; import apex.jorje.services.Version; @Deprecated @InternalApi -public abstract class ApexRootNode extends AbstractApexNode implements RootNode { +public abstract class ApexRootNode extends AbstractApexNode implements RootNode { @Deprecated @InternalApi public ApexRootNode(T node) { diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/AssignmentOperator.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/AssignmentOperator.java new file mode 100644 index 0000000000..6ee2bf6910 --- /dev/null +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/AssignmentOperator.java @@ -0,0 +1,67 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.apex.ast; + +import apex.jorje.data.ast.AssignmentOp; + +/** + * Apex assignment operator + */ +public enum AssignmentOperator { + EQUALS("="), + ADDITION_EQUALS("+="), + SUBTRACTION_EQUALS("-="), + MULTIPLICATION_EQUALS("*="), + DIVISION_EQUALS("/="), + LEFT_SHIFT_EQUALS("<<="), + RIGHT_SHIFT_SIGNED_EQUALS(">>="), + RIGHT_SHIFT_UNSIGNED_EQUALS(">>>="), + BITWISE_AND_EQUALS("&="), + BITWISE_OR_EQUALS("|="), + BITWISE_XOR_EQUALS("^="); + + private final String symbol; + + AssignmentOperator(String symbol) { + this.symbol = symbol; + } + + @Override + public String toString() { + return this.symbol; + } + + /** + * Returns a {@link AssignmentOperator} corresponding to the given {@link AssignmentOp}. + */ + public static AssignmentOperator valueOf(AssignmentOp op) { + switch (op) { + case EQUALS: + return EQUALS; + case ADDITION_EQUALS: + return ADDITION_EQUALS; + case SUBTRACTION_EQUALS: + return SUBTRACTION_EQUALS; + case MULTIPLICATION_EQUALS: + return MULTIPLICATION_EQUALS; + case DIVISION_EQUALS: + return DIVISION_EQUALS; + case LEFT_SHIFT_EQUALS: + return LEFT_SHIFT_EQUALS; + case RIGHT_SHIFT_EQUALS: + return RIGHT_SHIFT_SIGNED_EQUALS; + case UNSIGNED_RIGHT_SHIFT_EQUALS: + return RIGHT_SHIFT_UNSIGNED_EQUALS; + case AND_EQUALS: + return BITWISE_AND_EQUALS; + case OR_EQUALS: + return BITWISE_OR_EQUALS; + case XOR_EQUALS: + return BITWISE_XOR_EQUALS; + default: + throw new IllegalArgumentException("Invalid assignment operator " + op); + } + } +} diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BinaryOperator.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BinaryOperator.java new file mode 100644 index 0000000000..ea18dce06b --- /dev/null +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BinaryOperator.java @@ -0,0 +1,64 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.apex.ast; + +import apex.jorje.data.ast.BinaryOp; + +/** + * Apex binary operator + */ +public enum BinaryOperator { + ADDITION("+"), + SUBTRACTION("-"), + MULTIPLICATION("*"), + DIVISION("/"), + LEFT_SHIFT("<<"), + RIGHT_SHIFT_SIGNED(">>"), + RIGHT_SHIFT_UNSIGNED(">>>"), + BITWISE_AND("&"), + BITWISE_OR("|"), + BITWISE_XOR("^"); + + private final String symbol; + + BinaryOperator(String symbol) { + this.symbol = symbol; + } + + @Override + public String toString() { + return this.symbol; + } + + /** + * Returns a {@link BinaryOperator} corresponding to the given {@link BinaryOp}. + */ + public static BinaryOperator valueOf(BinaryOp op) { + switch (op) { + case ADDITION: + return ADDITION; + case SUBTRACTION: + return SUBTRACTION; + case MULTIPLICATION: + return MULTIPLICATION; + case DIVISION: + return DIVISION; + case LEFT_SHIFT: + return LEFT_SHIFT; + case RIGHT_SHIFT: + return RIGHT_SHIFT_SIGNED; + case UNSIGNED_RIGHT_SHIFT: + return RIGHT_SHIFT_UNSIGNED; + case AND: + return BITWISE_AND; + case OR: + return BITWISE_OR; + case XOR: + return BITWISE_XOR; + default: + throw new IllegalArgumentException("Invalid binary operator " + op); + } + } +} diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BooleanOperator.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BooleanOperator.java new file mode 100644 index 0000000000..e463eb2790 --- /dev/null +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/BooleanOperator.java @@ -0,0 +1,67 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.apex.ast; + +import apex.jorje.data.ast.BooleanOp; + +/** + * Apex boolean operator + */ +public enum BooleanOperator { + EQUAL("=="), + NOT_EQUAL("!="), + ALT_NOT_EQUAL("<>"), + EXACTLY_EQUAL("==="), + EXACTLY_NOT_EQUAL("!=="), + LESS_THAN("<"), + GREATER_THAN(">"), + LESS_THAN_OR_EQUAL("<="), + GREATER_THAN_OR_EQUAL(">="), + LOGICAL_AND("&&"), + LOGICAL_OR("||"); + + private final String symbol; + + BooleanOperator(String symbol) { + this.symbol = symbol; + } + + @Override + public String toString() { + return this.symbol; + } + + /** + * Returns a {@link BooleanOperator} corresponding to the given {@link BooleanOp}. + */ + public static BooleanOperator valueOf(BooleanOp op) { + switch (op) { + case DOUBLE_EQUAL: + return EQUAL; + case NOT_EQUAL: + return NOT_EQUAL; + case ALT_NOT_EQUAL: + return ALT_NOT_EQUAL; + case TRIPLE_EQUAL: + return EXACTLY_EQUAL; + case NOT_TRIPLE_EQUAL: + return EXACTLY_NOT_EQUAL; + case LESS_THAN: + return LESS_THAN; + case GREATER_THAN: + return GREATER_THAN; + case LESS_THAN_EQUAL: + return LESS_THAN_OR_EQUAL; + case GREATER_THAN_EQUAL: + return GREATER_THAN_OR_EQUAL; + case AND: + return LOGICAL_AND; + case OR: + return LOGICAL_OR; + default: + throw new IllegalArgumentException("Invalid boolean operator " + op); + } + } +} diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PostfixOperator.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PostfixOperator.java new file mode 100644 index 0000000000..5aa5a22756 --- /dev/null +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PostfixOperator.java @@ -0,0 +1,40 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.apex.ast; + +import apex.jorje.data.ast.PostfixOp; + +/** + * Apex postfix operator + */ +public enum PostfixOperator { + INCREMENT("++"), + DECREMENT("--"); + + private final String symbol; + + PostfixOperator(String symbol) { + this.symbol = symbol; + } + + @Override + public String toString() { + return this.symbol; + } + + /** + * Returns a {@link PostfixOperator} corresponding to the given {@link PostfixOp}. + */ + public static PostfixOperator valueOf(PostfixOp op) { + switch (op) { + case INC: + return INCREMENT; + case DEC: + return DECREMENT; + default: + throw new IllegalArgumentException("Invalid postfix operator " + op); + } + } +} diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PrefixOperator.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PrefixOperator.java new file mode 100644 index 0000000000..03c3b8379a --- /dev/null +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/PrefixOperator.java @@ -0,0 +1,52 @@ +/** + * BSD-style license; for more info see http://pmd.sourceforge.net/license.html + */ + +package net.sourceforge.pmd.lang.apex.ast; + +import apex.jorje.data.ast.PrefixOp; + +/** + * Apex prefix operator + */ +public enum PrefixOperator { + POSITIVE("+"), + NEGATIVE("-"), + LOGICAL_NOT("!"), + BITWISE_NOT("~"), + INCREMENT("++"), + DECREMENT("--"); + + private final String symbol; + + PrefixOperator(String symbol) { + this.symbol = symbol; + } + + @Override + public String toString() { + return this.symbol; + } + + /** + * Returns a {@link PrefixOperator} corresponding to the given {@link PrefixOp}. + */ + public static PrefixOperator valueOf(PrefixOp op) { + switch (op) { + case POSITIVE: + return POSITIVE; + case NEGATIVE: + return NEGATIVE; + case NOT: + return LOGICAL_NOT; + case BITWISE_COMPLEMENT: + return BITWISE_NOT; + case INC: + return INCREMENT; + case DEC: + return DECREMENT; + default: + throw new IllegalArgumentException("Invalid prefix operator " + op); + } + } +} diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/CycloMetric.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/CycloMetric.java index 6e60120f87..7c4f87de4a 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/CycloMetric.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/CycloMetric.java @@ -12,11 +12,10 @@ import org.apache.commons.lang3.mutable.MutableInt; import net.sourceforge.pmd.lang.apex.ast.ASTBooleanExpression; import net.sourceforge.pmd.lang.apex.ast.ASTMethod; import net.sourceforge.pmd.lang.apex.ast.ASTStandardCondition; +import net.sourceforge.pmd.lang.apex.ast.BooleanOperator; import net.sourceforge.pmd.lang.apex.metrics.impl.visitors.StandardCycloVisitor; import net.sourceforge.pmd.lang.metrics.MetricOptions; -import apex.jorje.data.ast.BooleanOp; - /** * See the doc for the Java metric. * @@ -44,8 +43,8 @@ public class CycloMetric extends AbstractApexOperationMetric { int complexity = 0; for (ASTBooleanExpression sub : subs) { - BooleanOp op = sub.getOperator(); - if (op != null && (op == BooleanOp.AND || op == BooleanOp.OR)) { + BooleanOperator op = sub.getOp(); + if (op == BooleanOperator.LOGICAL_AND || op == BooleanOperator.LOGICAL_OR) { complexity++; } } diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/visitors/CognitiveComplexityVisitor.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/visitors/CognitiveComplexityVisitor.java index 0fe1062d34..51b19aa9b1 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/visitors/CognitiveComplexityVisitor.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/metrics/impl/visitors/CognitiveComplexityVisitor.java @@ -19,9 +19,8 @@ import net.sourceforge.pmd.lang.apex.ast.ASTTernaryExpression; import net.sourceforge.pmd.lang.apex.ast.ASTWhileLoopStatement; import net.sourceforge.pmd.lang.apex.ast.ApexNode; import net.sourceforge.pmd.lang.apex.ast.ApexParserVisitorAdapter; - -import apex.jorje.data.ast.BooleanOp; -import apex.jorje.data.ast.PrefixOp; +import net.sourceforge.pmd.lang.apex.ast.BooleanOperator; +import net.sourceforge.pmd.lang.apex.ast.PrefixOperator; /** * @author Gwilym Kuiper @@ -31,7 +30,7 @@ public class CognitiveComplexityVisitor extends ApexParserVisitorAdapter { private int complexity = 0; private int nestingLevel = 0; - private BooleanOp currentBooleanOperation = null; + private BooleanOperator currentBooleanOperation = null; private String methodName = null; public double getComplexity() { @@ -53,7 +52,7 @@ public class CognitiveComplexityVisitor extends ApexParserVisitorAdapter { complexity++; } - void booleanOperation(BooleanOp op) { + void booleanOperation(BooleanOperator op) { if (currentBooleanOperation != op) { if (op != null) { fundamentalComplexity(); @@ -177,8 +176,8 @@ public class CognitiveComplexityVisitor extends ApexParserVisitorAdapter { public Object visit(ASTBooleanExpression node, Object data) { State state = (State) data; - BooleanOp op = node.getOperator(); - if (op == BooleanOp.AND || op == BooleanOp.OR) { + BooleanOperator op = node.getOp(); + if (op == BooleanOperator.LOGICAL_AND || op == BooleanOperator.LOGICAL_OR) { state.booleanOperation(op); } @@ -189,8 +188,8 @@ public class CognitiveComplexityVisitor extends ApexParserVisitorAdapter { public Object visit(ASTPrefixExpression node, Object data) { State state = (State) data; - PrefixOp op = node.getOperator(); - if (op == PrefixOp.NOT) { + PrefixOperator op = node.getOp(); + if (op == PrefixOperator.LOGICAL_NOT) { state.booleanOperation(null); } diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTFieldTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTFieldTest.java index b7cda1793c..1e8c2e1e44 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTFieldTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTFieldTest.java @@ -7,13 +7,11 @@ package net.sourceforge.pmd.lang.apex.ast; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTFieldTest extends ApexParserTestBase { @Test public void testGetType() { - ApexNode node = parse("public class Foo { private String myField = 'a'; }"); + ApexNode node = parse("public class Foo { private String myField = 'a'; }"); ASTField field = node.getFirstDescendantOfType(ASTField.class); Assert.assertEquals("myField", field.getImage()); @@ -23,7 +21,7 @@ public class ASTFieldTest extends ApexParserTestBase { @Test public void testGetValue() { - ApexNode node = parse("public class Foo { private String myField = 'a'; }"); + ApexNode node = parse("public class Foo { private String myField = 'a'; }"); ASTField field = node.getFirstDescendantOfType(ASTField.class); Assert.assertEquals("a", field.getValue()); @@ -31,7 +29,7 @@ public class ASTFieldTest extends ApexParserTestBase { @Test public void testGetNoValue() { - ApexNode node = parse("public class Foo { private String myField; }"); + ApexNode node = parse("public class Foo { private String myField; }"); ASTField field = node.getFirstDescendantOfType(ASTField.class); Assert.assertNull(field.getValue()); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTMethodTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTMethodTest.java index 832ab9e976..060b40a368 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTMethodTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTMethodTest.java @@ -9,13 +9,11 @@ import java.util.List; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTMethodTest extends ApexParserTestBase { @Test public void testConstructorName() { - ApexNode node = parse("public class Foo { public Foo() {} public void bar() {} }"); + ApexNode node = parse("public class Foo { public Foo() {} public void bar() {} }"); Assert.assertSame(ASTUserClass.class, node.getClass()); List methods = node.findChildrenOfType(ASTMethod.class); Assert.assertEquals("Foo", methods.get(0).getImage()); // constructor diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTNewKeyValueObjectExpressionTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTNewKeyValueObjectExpressionTest.java index 82fd256867..927c559790 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTNewKeyValueObjectExpressionTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTNewKeyValueObjectExpressionTest.java @@ -9,13 +9,11 @@ import java.util.List; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTNewKeyValueObjectExpressionTest extends ApexParserTestBase { @Test public void testParameterName() { - ApexNode node = parse("public class Foo { \n" + ApexNode node = parse("public class Foo { \n" + " public void foo(String newName, String tempID) { \n" + " if (Contact.sObjectType.getDescribe().isCreateable() && Contact.sObjectType.getDescribe().isUpdateable()) {\n" + " upsert new Contact(FirstName = 'First', LastName = 'Last', Phone = '414-414-4414');\n" diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSoqlExpressionTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSoqlExpressionTest.java index 4bf940aaa2..31a933ea2d 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSoqlExpressionTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSoqlExpressionTest.java @@ -7,13 +7,11 @@ package net.sourceforge.pmd.lang.apex.ast; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTSoqlExpressionTest extends ApexParserTestBase { @Test public void testQuery() { - ApexNode node = parse("class Foo { void test1() { Account acc = [SELECT col FROM Account]; } }"); + ApexNode node = parse("class Foo { void test1() { Account acc = [SELECT col FROM Account]; } }"); ASTSoqlExpression soqlExpression = node.getFirstDescendantOfType(ASTSoqlExpression.class); Assert.assertEquals("SELECT col FROM Account", soqlExpression.getQuery()); } diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSwitchStatementTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSwitchStatementTest.java index 72623c8ecc..f5dddb17c0 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSwitchStatementTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTSwitchStatementTest.java @@ -9,13 +9,11 @@ import java.util.List; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTSwitchStatementTest extends ApexParserTestBase { @Test public void testExamples() { - ApexNode node = parseResource("SwitchStatements.cls"); + ApexNode node = parseResource("SwitchStatements.cls"); List switchStatements = node.findDescendantsOfType(ASTSwitchStatement.class); Assert.assertEquals(4, switchStatements.size()); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTTryCatchFinallyBlockStatementTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTTryCatchFinallyBlockStatementTest.java index a45ac3dfb8..a6cd4df9b9 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTTryCatchFinallyBlockStatementTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTTryCatchFinallyBlockStatementTest.java @@ -7,13 +7,11 @@ package net.sourceforge.pmd.lang.apex.ast; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTTryCatchFinallyBlockStatementTest extends ApexParserTestBase { @Test public void testTryFinally() { - ApexNode node = parse("class Foo { void bar() { try { methodCall(); } finally { methodCall(); } } }"); + ApexNode node = parse("class Foo { void bar() { try { methodCall(); } finally { methodCall(); } } }"); ASTTryCatchFinallyBlockStatement statement = node.getFirstDescendantOfType(ASTTryCatchFinallyBlockStatement.class); Assert.assertNotNull(statement.getTryBlock()); Assert.assertEquals(0, statement.getTryBlock().getIndexInParent()); @@ -24,7 +22,7 @@ public class ASTTryCatchFinallyBlockStatementTest extends ApexParserTestBase { @Test public void testTryCatch() { - ApexNode node = parse("class Foo { void bar() { try { methodCall(); } catch (Exception e) { methodCall(); } } }"); + ApexNode node = parse("class Foo { void bar() { try { methodCall(); } catch (Exception e) { methodCall(); } } }"); ASTTryCatchFinallyBlockStatement statement = node.getFirstDescendantOfType(ASTTryCatchFinallyBlockStatement.class); Assert.assertNotNull(statement.getTryBlock()); Assert.assertEquals(0, statement.getTryBlock().getIndexInParent()); @@ -36,7 +34,7 @@ public class ASTTryCatchFinallyBlockStatementTest extends ApexParserTestBase { @Test public void testTryCatchFinally() { - ApexNode node = parse("class Foo { void bar() { try { methodCall(); } catch (Exception e) { methodCall(); } finally { } } }"); + ApexNode node = parse("class Foo { void bar() { try { methodCall(); } catch (Exception e) { methodCall(); } finally { } } }"); ASTTryCatchFinallyBlockStatement statement = node.getFirstDescendantOfType(ASTTryCatchFinallyBlockStatement.class); Assert.assertNotNull(statement.getTryBlock()); Assert.assertEquals(0, statement.getTryBlock().getIndexInParent()); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserClassTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserClassTest.java index 4fa039f9ce..9a2cda1265 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserClassTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserClassTest.java @@ -9,20 +9,18 @@ import java.util.Arrays; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTUserClassTest extends ApexParserTestBase { @Test public void testClassName() { - ApexNode node = parse("class Foo { }"); + ApexNode node = parse("class Foo { }"); Assert.assertSame(ASTUserClass.class, node.getClass()); Assert.assertEquals("Foo", node.getImage()); } @Test public void testInnerClassName() { - ApexNode node = parse("class Foo { class Bar { } }"); + ApexNode node = parse("class Foo { class Bar { } }"); Assert.assertSame(ASTUserClass.class, node.getClass()); ASTUserClass innerNode = node.getFirstDescendantOfType(ASTUserClass.class); Assert.assertNotNull(innerNode); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnumTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnumTest.java index cf67095019..c03125ecb5 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnumTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnumTest.java @@ -7,13 +7,11 @@ package net.sourceforge.pmd.lang.apex.ast; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTUserEnumTest extends ApexParserTestBase { @Test public void testEnumName() { - ApexNode node = parse("class Foo { enum Bar { } }"); + ApexNode node = parse("class Foo { enum Bar { } }"); Assert.assertSame(ASTUserClass.class, node.getClass()); ASTUserEnum enumNode = node.getFirstDescendantOfType(ASTUserEnum.class); Assert.assertNotNull(enumNode); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserInterfaceTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserInterfaceTest.java index a53124139b..6b56d677b9 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserInterfaceTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ASTUserInterfaceTest.java @@ -7,20 +7,18 @@ package net.sourceforge.pmd.lang.apex.ast; import org.junit.Assert; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ASTUserInterfaceTest extends ApexParserTestBase { @Test public void testInterfaceName() { - ApexNode node = parse("interface Foo { }"); + ApexNode node = parse("interface Foo { }"); Assert.assertSame(ASTUserInterface.class, node.getClass()); Assert.assertEquals("Foo", node.getImage()); } @Test public void testInnerInterfaceName() { - ApexNode node = parse("class Foo { interface Bar { } }"); + ApexNode node = parse("class Foo { interface Bar { } }"); Assert.assertSame(ASTUserClass.class, node.getClass()); ASTUserInterface innerNode = node.getFirstDescendantOfType(ASTUserInterface.class); Assert.assertNotNull(innerNode); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTest.java index be3f11d57e..10c7875243 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTest.java @@ -21,8 +21,6 @@ import net.sourceforge.pmd.lang.ast.Node; import net.sourceforge.pmd.lang.ast.xpath.internal.FileNameXPathFunction; import net.sourceforge.pmd.util.IOUtil; -import apex.jorje.semantic.ast.compilation.Compilation; - public class ApexParserTest extends ApexParserTestBase { @Test @@ -33,7 +31,7 @@ public class ApexParserTest extends ApexParserTestBase { + " \n" + " }\n" + "}"; // Exercise - ApexNode rootNode = parse(code); + ApexNode rootNode = parse(code); // Verify List methods = rootNode.findDescendantsOfType(ASTMethod.class); @@ -59,18 +57,18 @@ public class ApexParserTest extends ApexParserTestBase { @Test public void verifyLineColumnNumbers() { - ApexNode rootNode = parse(testCodeForLineNumbers); + ApexNode rootNode = parse(testCodeForLineNumbers); assertLineNumbersForTestCode(rootNode); } @Test public void verifyLineColumnNumbersWithWindowsLineEndings() { String windowsLineEndings = testCodeForLineNumbers.replaceAll(" \n", "\r\n"); - ApexNode rootNode = parse(windowsLineEndings); + ApexNode rootNode = parse(windowsLineEndings); assertLineNumbersForTestCode(rootNode); } - private void assertLineNumbersForTestCode(ApexNode rootNode) { + private void assertLineNumbersForTestCode(ApexNode rootNode) { // whole source code, well from the beginning of the class // name Modifier of the class - doesn't work. This node just // sees the identifier ("SimpleClass") @@ -107,7 +105,7 @@ public class ApexParserTest extends ApexParserTestBase { + " }\n" // line 5 + "}\n"; // line 6 - ApexNode rootNode = parse(code); + ApexNode rootNode = parse(code); Node method1 = rootNode.getChild(1); assertEquals("Wrong begin line", 2, method1.getBeginLine()); @@ -129,7 +127,7 @@ public class ApexParserTest extends ApexParserTestBase { + " }\n" // line 5 + "}\n"; // line 6 - ApexNode root = parse(code); + ApexNode root = parse(code); assertThat(root, instanceOf(ASTUserClass.class)); ApexNode comment = root.getChild(0); @@ -154,7 +152,7 @@ public class ApexParserTest extends ApexParserTestBase { for (File file : fList) { if (file.isFile() && file.getName().endsWith(".cls")) { String sourceCode = IOUtil.readFileToString(file, StandardCharsets.UTF_8); - ApexNode rootNode = parse(sourceCode); + ApexNode rootNode = parse(sourceCode); Assert.assertNotNull(rootNode); } } @@ -168,7 +166,7 @@ public class ApexParserTest extends ApexParserTestBase { public void parseInheritedSharingClass() throws IOException { String source = IOUtil.readToString(ApexParserTest.class.getResourceAsStream("InheritedSharing.cls"), StandardCharsets.UTF_8); - ApexNode rootNode = parse(source); + ApexNode rootNode = parse(source); Assert.assertNotNull(rootNode); } @@ -181,7 +179,7 @@ public class ApexParserTest extends ApexParserTestBase { public void stackOverflowDuringClassParsing() throws Exception { String source = IOUtil.readToString(ApexParserTest.class.getResourceAsStream("StackOverflowClass.cls"), StandardCharsets.UTF_8); - ApexNode rootNode = parse(source); + ApexNode rootNode = parse(source); Assert.assertNotNull(rootNode); int count = visitPosition(rootNode, 0); @@ -193,7 +191,7 @@ public class ApexParserTest extends ApexParserTestBase { String source = IOUtil.readToString(ApexParserTest.class.getResourceAsStream("InnerClassLocations.cls"), StandardCharsets.UTF_8); source = source.replaceAll("\r\n", "\n"); - ApexNode rootNode = parse(source); + ApexNode rootNode = parse(source); Assert.assertNotNull(rootNode); visitPosition(rootNode, 0); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTestBase.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTestBase.java index 728e57b3a9..ab27f77abb 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTestBase.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexParserTestBase.java @@ -11,15 +11,15 @@ public class ApexParserTestBase { protected final ApexParsingHelper apex = ApexParsingHelper.DEFAULT.withResourceContext(getClass()); - protected ApexNode parse(String code) { + protected ApexRootNode parse(String code) { return apex.parse(code); } - protected ApexNode parse(String code, String fileName) { + protected ApexRootNode parse(String code, String fileName) { return apex.parse(code, null, fileName); } - protected ApexNode parseResource(String code) { + protected ApexRootNode parseResource(String code) { return apex.parseResource(code); } } diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java index 515b830457..cf55cb8cc5 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java @@ -13,9 +13,6 @@ import java.util.List; import org.junit.Test; -import apex.jorje.semantic.ast.compilation.Compilation; - - /** * @author ClΓ©ment Fournier */ @@ -23,9 +20,9 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { @Test public void testClass() { - ApexNode root = parse("public class Foo {}"); + ApexNode root = parse("public class Foo {}"); - ApexQualifiedName qname = ASTUserClass.class.cast(root).getQualifiedName(); + ApexQualifiedName qname = ((ASTUserClass) root).getQualifiedName(); assertEquals("c__Foo", qname.toString()); assertEquals(1, qname.getClasses().length); assertNotNull(qname.getNameSpace()); @@ -35,7 +32,7 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { @Test public void testNestedClass() { - ApexNode root = parse("public class Foo { class Bar {}}"); + ApexNode root = parse("public class Foo { class Bar {}}"); ApexQualifiedName qname = root.getFirstDescendantOfType(ASTUserClass.class).getQualifiedName(); assertEquals("c__Foo.Bar", qname.toString()); @@ -47,7 +44,7 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { @Test public void testSimpleMethod() { - ApexNode root = parse("public class Foo { String foo() {}}"); + ApexNode root = parse("public class Foo { String foo() {}}"); ApexQualifiedName qname = root.getFirstDescendantOfType(ASTMethod.class).getQualifiedName(); assertEquals("c__Foo#foo()", qname.toString()); assertEquals(1, qname.getClasses().length); @@ -58,7 +55,7 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { @Test public void testMethodWithArguments() { - ApexNode root = parse("public class Foo { String foo(String h, Foo g) {}}"); + ApexNode root = parse("public class Foo { String foo(String h, Foo g) {}}"); ApexQualifiedName qname = root.getFirstDescendantOfType(ASTMethod.class).getQualifiedName(); assertEquals("c__Foo#foo(String, Foo)", qname.toString()); assertEquals(1, qname.getClasses().length); @@ -69,7 +66,7 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { @Test public void testOverLoads() { - ApexNode root = parse("public class Foo { " + ApexNode root = parse("public class Foo { " + "String foo(String h) {} " + "String foo(int c) {}" + "String foo(Foo c) {}}"); @@ -88,7 +85,7 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { @Test public void testTrigger() { - ApexNode root = parse("trigger myAccountTrigger on Account (before insert, before update) {}"); + ApexNode root = parse("trigger myAccountTrigger on Account (before insert, before update) {}"); List methods = root.findDescendantsOfType(ASTMethod.class); diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/metrics/ApexProjectMirrorTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/metrics/ApexProjectMirrorTest.java index 204bc20b7c..0c6cabf7b1 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/metrics/ApexProjectMirrorTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/metrics/ApexProjectMirrorTest.java @@ -27,14 +27,12 @@ import net.sourceforge.pmd.lang.metrics.MetricKeyUtil; import net.sourceforge.pmd.lang.metrics.MetricOptions; import net.sourceforge.pmd.lang.metrics.MetricsUtil; -import apex.jorje.semantic.ast.compilation.Compilation; - /** * @author ClΓ©ment Fournier */ public class ApexProjectMirrorTest extends ApexParserTestBase { - private static ApexNode acu; + private static ApexNode acu; private MetricKey> classMetricKey = MetricKeyUtil.of(null, new RandomClassMetric()); private MetricKey opMetricKey = MetricKeyUtil.of(null, new RandomOperationMetric()); @@ -70,7 +68,7 @@ public class ApexProjectMirrorTest extends ApexParserTestBase { } - private List visitWith(ApexNode acu, final boolean force) { + private List visitWith(ApexNode acu, final boolean force) { final List result = new ArrayList<>(); acu.jjtAccept(new ApexParserVisitorAdapter() { diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/multifile/ApexMultifileVisitorTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/multifile/ApexMultifileVisitorTest.java index 625a7d40eb..ce5829028b 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/multifile/ApexMultifileVisitorTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/multifile/ApexMultifileVisitorTest.java @@ -18,8 +18,6 @@ import net.sourceforge.pmd.lang.apex.ast.ApexParserVisitorAdapter; import net.sourceforge.pmd.lang.apex.metrics.ApexSignatureMatcher; import net.sourceforge.pmd.lang.apex.metrics.signature.ApexOperationSigMask; -import apex.jorje.semantic.ast.compilation.Compilation; - /** * @author ClΓ©ment Fournier */ @@ -33,7 +31,7 @@ public class ApexMultifileVisitorTest extends ApexParserTestBase { @Test public void testOperationsAreThere() throws IOException { - ApexNode acu = parseResource("MetadataDeployController.cls"); + ApexNode acu = parseResource("MetadataDeployController.cls"); final ApexSignatureMatcher toplevel = ApexProjectMirror.INSTANCE; diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/rule/AbstractApexRuleTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/rule/AbstractApexRuleTest.java index e0d0bc2737..6d7f80ce3e 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/rule/AbstractApexRuleTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/rule/AbstractApexRuleTest.java @@ -19,8 +19,6 @@ import net.sourceforge.pmd.lang.apex.ast.ASTUserTrigger; import net.sourceforge.pmd.lang.apex.ast.ApexNode; import net.sourceforge.pmd.lang.apex.ast.ApexParserTestBase; -import apex.jorje.semantic.ast.compilation.Compilation; - public class AbstractApexRuleTest extends ApexParserTestBase { @Test @@ -44,7 +42,7 @@ public class AbstractApexRuleTest extends ApexParserTestBase { } private void run(String code) { - ApexNode node = parse(code); + ApexNode node = parse(code); TopLevelRule rule = new TopLevelRule(); RuleContext ctx = new RuleContext(); ctx.setLanguageVersion(apex.getDefaultVersion()); From fbeb071d081f3ef848d8de34a7dcf32c7975b458 Mon Sep 17 00:00:00 2001 From: Aaron Hurst Date: Fri, 12 Aug 2022 19:10:54 +0000 Subject: [PATCH 090/136] Implement ApexQualifiableNode for ASTUserEnum. Include enum in qualified name. Properly handle case where enum is root node. Fixes null deref in ApexQualifiedName.toString() for built-in ASTMethods in enum types. Reference: https://developer.salesforce.com/docs/atlas.en-us.apexref.meta/apexref/apex_methods_system_enum.htm Add tests of qualified and unqualified enums. Change-Id: I691795d40a66f3d3335ab72ad43de7055a6aee31 --- .../pmd/lang/apex/ast/ASTUserEnum.java | 20 ++++++++- .../pmd/lang/apex/ast/ApexQualifiedName.java | 44 ++++++++++++++----- .../lang/apex/ast/ApexQualifiedNameTest.java | 29 ++++++++++++ 3 files changed, 82 insertions(+), 11 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java index 4aef1641cc..b4db014ea9 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java @@ -8,7 +8,9 @@ import net.sourceforge.pmd.annotation.InternalApi; import apex.jorje.semantic.ast.compilation.UserEnum; -public class ASTUserEnum extends ApexRootNode { +public class ASTUserEnum extends ApexRootNode implements ApexQualifiableNode { + + private ApexQualifiedName qname; @Deprecated @InternalApi @@ -30,4 +32,20 @@ public class ASTUserEnum extends ApexRootNode { public ASTModifierNode getModifiers() { return getFirstChildOfType(ASTModifierNode.class); } + + @Override + public ApexQualifiedName getQualifiedName() { + if (qname == null) { + + ASTUserClass parent = this.getFirstParentOfType(ASTUserClass.class); + + if (parent != null) { + qname = ApexQualifiedName.ofNestedEnum(parent.getQualifiedName(), this); + } else { + qname = ApexQualifiedName.ofOuterEnum(this); + } + } + + return qname; + } } diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedName.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedName.java index d3898a29fb..b5ce290e4a 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedName.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedName.java @@ -148,6 +148,20 @@ public final class ApexQualifiedName implements QualifiedName { } + static ApexQualifiedName ofOuterEnum(ASTUserEnum astUserEnum) { + String ns = astUserEnum.getNamespace(); + String[] classes = {astUserEnum.getImage()}; + return new ApexQualifiedName(StringUtils.isEmpty(ns) ? "c" : ns, classes, null); + } + + + static ApexQualifiedName ofNestedEnum(ApexQualifiedName parent, ASTUserEnum astUserEnum) { + String[] classes = Arrays.copyOf(parent.classes, parent.classes.length + 1); + classes[classes.length - 1] = astUserEnum.getImage(); + return new ApexQualifiedName(parent.nameSpace, classes, null); + } + + private static String getOperationString(ASTMethod node) { StringBuilder sb = new StringBuilder(); sb.append(node.getImage()).append('('); @@ -171,18 +185,28 @@ public final class ApexQualifiedName implements QualifiedName { static ApexQualifiedName ofMethod(ASTMethod node) { - ASTUserClassOrInterface parent = node.getFirstParentOfType(ASTUserClassOrInterface.class); - if (parent == null) { - ASTUserTrigger trigger = node.getFirstParentOfType(ASTUserTrigger.class); - String ns = trigger.getNamespace(); - String targetObj = trigger.getTargetName(); - - return new ApexQualifiedName(StringUtils.isEmpty(ns) ? "c" : ns, new String[]{"trigger", targetObj}, trigger.getImage()); // uses a reserved word as a class name to prevent clashes - - } else { - ApexQualifiedName baseName = parent.getQualifiedName(); + // Check first, as enum must be innermost potential parent + ASTUserEnum enumParent = node.getFirstParentOfType(ASTUserEnum.class); + if (enumParent != null) { + ApexQualifiedName baseName = enumParent.getQualifiedName(); return new ApexQualifiedName(baseName.nameSpace, baseName.classes, getOperationString(node)); } + + ASTUserClassOrInterface classParent = node.getFirstParentOfType(ASTUserClassOrInterface.class); + if (classParent != null) { + ApexQualifiedName baseName = classParent.getQualifiedName(); + + return new ApexQualifiedName(baseName.nameSpace, baseName.classes, getOperationString(node)); + } + + ASTUserTrigger triggerParent = node.getFirstParentOfType(ASTUserTrigger.class); + if (triggerParent != null) { + String ns = triggerParent.getNamespace(); + String targetObj = triggerParent.getTargetName(); + + return new ApexQualifiedName(StringUtils.isEmpty(ns) ? "c" : ns, new String[]{"trigger", targetObj}, triggerParent.getImage()); // uses a reserved word as a class name to prevent clashes + } + throw new UnsupportedOperationException(); } } diff --git a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java index 515b830457..d290a4044a 100644 --- a/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java +++ b/pmd-apex/src/test/java/net/sourceforge/pmd/lang/apex/ast/ApexQualifiedNameTest.java @@ -8,6 +8,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; import java.util.List; @@ -97,4 +98,32 @@ public class ApexQualifiedNameTest extends ApexParserTestBase { assertEquals("c__trigger.Account#myAccountTrigger", m.getQualifiedName().toString()); } } + + + @Test + public void testUnqualifiedEnum() { + ApexNode root = parse("public enum primaryColor { RED, YELLOW, BLUE }"); + + ApexQualifiedName enumQName = ASTUserEnum.class.cast(root).getQualifiedName(); + List methods = root.findDescendantsOfType(ASTMethod.class); + + assertEquals("c__primaryColor", enumQName.toString()); + for (ASTMethod m : methods) { + assertTrue(m.getQualifiedName().toString().startsWith("c__primaryColor#")); + } + } + + @Test + public void testQualifiedEnum() { + ApexNode root = parse("public class Outer { public enum Inner { OK } }"); + + ASTUserEnum enumNode = root.getFirstDescendantOfType(ASTUserEnum.class); + ApexQualifiedName enumQName = enumNode.getQualifiedName(); + List methods = enumNode.findDescendantsOfType(ASTMethod.class); + + assertEquals("c__Outer.Inner", enumQName.toString()); + for (ASTMethod m : methods) { + assertTrue(m.getQualifiedName().toString().startsWith("c__Outer.Inner#")); + } + } } From 91fbcc804abf5cef726bb44eab1bd92703a5eb64 Mon Sep 17 00:00:00 2001 From: zon <> Date: Wed, 17 Aug 2022 12:40:39 +0200 Subject: [PATCH 091/136] Added begin and end token attributes to XML output of CPD --- .../pmd/userdocs/cpd/cpd_report_formats.md | 10 +++--- .../java/net/sourceforge/pmd/cpd/Mark.java | 8 +++++ .../net/sourceforge/pmd/cpd/XMLRenderer.java | 6 ++++ .../sourceforge/pmd/cpd/XMLRendererTest.java | 32 +++++++++++++++++++ 4 files changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md b/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md index 4e1a23a0d0..f21bd5b448 100644 --- a/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md +++ b/docs/pages/pmd/userdocs/cpd/cpd_report_formats.md @@ -104,9 +104,9 @@ Example: - - - - - ruleChainVisits = query.getRuleChainVisits(); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java index c741a337ad..30e13c4b04 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java @@ -35,6 +35,10 @@ public class Mark implements Comparable { return this.token.getBeginColumn(); // TODO Java 1.8 make optional } + public int getBeginTokenIndex() { + return this.token.getIndex(); + } + public int getEndLine() { return getBeginLine() + getLineCount() - 1; } @@ -48,6 +52,10 @@ public class Mark implements Comparable { return this.endToken == null ? -1 : this.endToken.getEndColumn(); // TODO Java 1.8 make optional } + public int getEndTokenIndex() { + return this.endToken == null ? -1 : this.endToken.getIndex(); + } + public int getLineCount() { return this.lineCount; } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java index d7d69382e5..6b5d51a2cf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/XMLRenderer.java @@ -152,6 +152,12 @@ public final class XMLRenderer implements Renderer, CPDRenderer, CPDReportRender if (endCol != -1) { file.setAttribute("endcolumn", String.valueOf(endCol)); } + final int beginIndex = mark.getBeginTokenIndex(); + final int endIndex = mark.getEndTokenIndex(); + file.setAttribute("begintoken", String.valueOf(beginIndex)); + if (endIndex != -1) { + file.setAttribute("endtoken", String.valueOf(endIndex)); + } duplication.appendChild(file); } return duplication; diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java index 1e29e7961b..366733b088 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/cpd/XMLRendererTest.java @@ -216,6 +216,38 @@ public class XMLRendererTest { assertEquals("888", attributes.getNamedItem("totalNumberOfTokens").getNodeValue()); } + @Test + public void testGetDuplicationStartEnd() throws IOException, ParserConfigurationException, SAXException { + TokenEntry.clearImages(); + final CPDReportRenderer renderer = new XMLRenderer(); + final List matches = new ArrayList<>(); + final String filename = "/var/Foo.java"; + final int lineCount = 6; + final String codeFragment = "code\nfragment"; + final Mark mark1 = createMark("public", filename, 1, lineCount, codeFragment, 2, 3); + final Mark mark2 = createMark("stuff", filename, 73, lineCount, codeFragment, 4, 5); + final Match match = new Match(75, mark1, mark2); + matches.add(match); + final Map numberOfTokensPerFile = new HashMap<>(); + numberOfTokensPerFile.put(filename, 888); + final CPDReport report = new CPDReport(matches, numberOfTokensPerFile); + final StringWriter writer = new StringWriter(); + renderer.render(report, writer); + final String xmlOutput = writer.toString(); + final Document doc = DocumentBuilderFactory.newInstance().newDocumentBuilder() + .parse(new ByteArrayInputStream(xmlOutput.getBytes(ENCODING))); + final NodeList files = doc.getElementsByTagName("file"); + final Node dup_1 = files.item(1); + final NamedNodeMap attrs_1 = dup_1.getAttributes(); + assertEquals("0", attrs_1.getNamedItem("begintoken").getNodeValue()); + assertEquals("1", attrs_1.getNamedItem("endtoken").getNodeValue()); + + final Node dup_2 = files.item(2); + final NamedNodeMap attrs_2 = dup_2.getAttributes(); + assertEquals("2", attrs_2.getNamedItem("begintoken").getNodeValue()); + assertEquals("3", attrs_2.getNamedItem("endtoken").getNodeValue()); + } + @Test public void testRendererEncodedPath() throws IOException { CPDRenderer renderer = new XMLRenderer(); From 237b1066760b992c449a23fdcbf28677dd4f0c68 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Wed, 17 Aug 2022 15:09:54 +0200 Subject: [PATCH 092/136] Add new assert methods for api 56 --- ...ApexUnitTestClassShouldHaveAssertsRule.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java index 084a25daf6..083ea31356 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java @@ -37,6 +37,24 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest ASSERT_METHODS.add("system.assert"); ASSERT_METHODS.add("system.assertequals"); ASSERT_METHODS.add("system.assertnotequals"); + ASSERT_METHODS.add("system.assert.areequal"); + ASSERT_METHODS.add("system.assert.arenotequal"); + ASSERT_METHODS.add("system.assert.fail"); + ASSERT_METHODS.add("system.assert.isfalse"); + ASSERT_METHODS.add("system.assert.isinstanceoftype"); + ASSERT_METHODS.add("system.assert.isnotinstanceoftype"); + ASSERT_METHODS.add("system.assert.isnnull"); + ASSERT_METHODS.add("system.assert.isnotnull"); + ASSERT_METHODS.add("system.assert.istrue"); + ASSERT_METHODS.add("assert.areequal"); + ASSERT_METHODS.add("assert.arenotequal"); + ASSERT_METHODS.add("assert.fail"); + ASSERT_METHODS.add("assert.isfalse"); + ASSERT_METHODS.add("assert.isinstanceoftype"); + ASSERT_METHODS.add("assert.isnotinstanceoftype"); + ASSERT_METHODS.add("assert.isnnull"); + ASSERT_METHODS.add("assert.isnotnull"); + ASSERT_METHODS.add("assert.istrue"); // Fully-qualified variants...rare but still valid/possible ASSERT_METHODS.add("system.system.assert"); ASSERT_METHODS.add("system.system.assertequals"); From 2e1d913058be17c0407203abd6f8bf5d4e47e427 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 18 Aug 2022 10:27:06 +0200 Subject: [PATCH 093/136] Update @tprouvot as a contributor --- .all-contributorsrc | 3 ++- docs/pages/pmd/projectdocs/credits.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 99708783e8..4b24a733c3 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -5917,7 +5917,8 @@ "avatar_url": "https://avatars.githubusercontent.com/u/35368290?v=4", "profile": "https://github.com/tprouvot", "contributions": [ - "bug" + "bug", + "code" ] }, { diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 37788cf5f7..eb60d9190c 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -932,7 +932,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
tobwoerk
πŸ› -
tprouvot
πŸ› +
tprouvot
πŸ› πŸ’»
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ› From 2edcfe25cdbab7c56ec0df1ed89fac7a3110d6ac Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 18 Aug 2022 10:28:19 +0200 Subject: [PATCH 094/136] Update @tprouvot as a contributor --- .all-contributorsrc | 1 - docs/pages/pmd/projectdocs/credits.md | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 4b24a733c3..5fba86143b 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -5917,7 +5917,6 @@ "avatar_url": "https://avatars.githubusercontent.com/u/35368290?v=4", "profile": "https://github.com/tprouvot", "contributions": [ - "bug", "code" ] }, diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index eb60d9190c..00199aea11 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -932,7 +932,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
tobwoerk
πŸ› -
tprouvot
πŸ› πŸ’» +
tprouvot
πŸ’»
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ› From 539838c9064b377df16e2b7dced257790bc6e3c4 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 18 Aug 2022 14:40:50 +0200 Subject: [PATCH 095/136] Update previous negative test which is ok now and add new positive test. --- .../xml/ApexUnitTestClassShouldHaveAsserts.xml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index 8e9520f15d..69d0cba08f 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -103,7 +103,7 @@ public class Foo { @isTest public class Foo { public static testMethod void testAssertIsTrue() { - Assert.isTrue(someCondition); + Assert.assertEquals(someCondition); } public static testMethod void testLocalVerify() { @@ -115,4 +115,18 @@ public class Foo { } ]]> + + + #4096 [apex] api 56.0 ApexAssertionsShouldIncludeMessage and new apex class : Assert + 0 + + From ad21a24b74f96c1e5df0fb7c5075c7f8079c18de Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 18 Aug 2022 14:48:25 +0200 Subject: [PATCH 096/136] Fix contributors --- .all-contributorsrc | 4 ++-- docs/pages/pmd/projectdocs/credits.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 5fba86143b..16274b1839 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -5917,7 +5917,7 @@ "avatar_url": "https://avatars.githubusercontent.com/u/35368290?v=4", "profile": "https://github.com/tprouvot", "contributions": [ - "code" + "bug" ] }, { @@ -6773,4 +6773,4 @@ "contributorsPerLine": 7, "contributorsSortAlphabetically": true, "skipCi": true -} +} \ No newline at end of file diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 00199aea11..8a7c21410f 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -932,7 +932,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
tobwoerk
πŸ› -
tprouvot
πŸ’» +
tprouvot
πŸ›
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ› From a2058c454b551535b243e53553ee7e4370563d3d Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 18 Aug 2022 16:34:14 +0200 Subject: [PATCH 097/136] move fully qualified variant under comment (PR Comment) --- ...ApexUnitTestClassShouldHaveAssertsRule.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java index 083ea31356..2e0d717b60 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java @@ -37,15 +37,6 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest ASSERT_METHODS.add("system.assert"); ASSERT_METHODS.add("system.assertequals"); ASSERT_METHODS.add("system.assertnotequals"); - ASSERT_METHODS.add("system.assert.areequal"); - ASSERT_METHODS.add("system.assert.arenotequal"); - ASSERT_METHODS.add("system.assert.fail"); - ASSERT_METHODS.add("system.assert.isfalse"); - ASSERT_METHODS.add("system.assert.isinstanceoftype"); - ASSERT_METHODS.add("system.assert.isnotinstanceoftype"); - ASSERT_METHODS.add("system.assert.isnnull"); - ASSERT_METHODS.add("system.assert.isnotnull"); - ASSERT_METHODS.add("system.assert.istrue"); ASSERT_METHODS.add("assert.areequal"); ASSERT_METHODS.add("assert.arenotequal"); ASSERT_METHODS.add("assert.fail"); @@ -59,6 +50,15 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest ASSERT_METHODS.add("system.system.assert"); ASSERT_METHODS.add("system.system.assertequals"); ASSERT_METHODS.add("system.system.assertnotequals"); + ASSERT_METHODS.add("system.assert.areequal"); + ASSERT_METHODS.add("system.assert.arenotequal"); + ASSERT_METHODS.add("system.assert.fail"); + ASSERT_METHODS.add("system.assert.isfalse"); + ASSERT_METHODS.add("system.assert.isinstanceoftype"); + ASSERT_METHODS.add("system.assert.isnotinstanceoftype"); + ASSERT_METHODS.add("system.assert.isnnull"); + ASSERT_METHODS.add("system.assert.isnotnull"); + ASSERT_METHODS.add("system.assert.istrue"); } // Using a string property instead of a regex property to ensure that the compiled pattern can be case-insensitive From dc03e52be3c951a0430135fa5c29dca1a57d1a29 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 18 Aug 2022 16:34:46 +0200 Subject: [PATCH 098/136] Update method name and number of parameters (PR Comment) --- .../bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index 69d0cba08f..b78a6c86f7 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -102,8 +102,8 @@ public class Foo { Date: Thu, 18 Aug 2022 17:11:22 +0200 Subject: [PATCH 099/136] fix contributors --- .all-contributorsrc | 2 +- docs/pages/pmd/projectdocs/credits.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 16274b1839..99708783e8 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -6773,4 +6773,4 @@ "contributorsPerLine": 7, "contributorsSortAlphabetically": true, "skipCi": true -} \ No newline at end of file +} diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 8a7c21410f..37788cf5f7 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -932,7 +932,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
tobwoerk
πŸ› -
tprouvot
πŸ› +
tprouvot
πŸ›
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ› From 8fcbcb434e9e2862989e0fbd0ecd4e3c0250dac2 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 17:27:34 +0200 Subject: [PATCH 100/136] [doc] Update release notes (#4082, #4083) --- docs/pages/release_notes.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..f14ef9960d 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,9 +16,14 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* java-codestyle + * [#4082](https://github.com/pmd/pmd/issues/4082): \[java] UnnecessaryImport false positive for on-demand imports of nested classes + ### API Changes ### External Contributions +* [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) + {% endtocmaker %} From 075507cb0183aa09256508c1a7d3dc311c79071b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 17:28:13 +0200 Subject: [PATCH 101/136] Add @abyss638 as a contributor --- .all-contributorsrc | 9 ++++ docs/pages/pmd/projectdocs/credits.md | 75 ++++++++++++++------------- 2 files changed, 47 insertions(+), 37 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 99708783e8..9ef6610038 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -6768,6 +6768,15 @@ "code", "financial" ] + }, + { + "login": "abyss638", + "name": "Simon Abykov", + "avatar_url": "https://avatars.githubusercontent.com/u/90252673?v=4", + "profile": "https://github.com/abyss638", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 37788cf5f7..73955ce6f7 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -632,333 +632,334 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Sergey Yanzin
πŸ’» πŸ›
Seth Wilcox
πŸ’»
Shubham
πŸ’» πŸ› -
Simon Xiao
πŸ› +
Simon Abykov
πŸ’» +
Simon Xiao
πŸ›
Srinivasan Venkatachalam
πŸ›
Stanislav Gromov
πŸ›
Stanislav Myachenkov
πŸ’»
Stefan Birkner
πŸ›
Stefan Bohn
πŸ›
Stefan Endrullis
πŸ› -
Stefan KlΓΆss-Schuster
πŸ› +
Stefan KlΓΆss-Schuster
πŸ›
Stefan Wolf
πŸ›
Stephan H. Wissel
πŸ›
Stephen
πŸ›
Stephen Friedrich
πŸ›
Steve Babula
πŸ’»
Stexxe
πŸ› -
Stian LΓ₯gstad
πŸ› +
Stian LΓ₯gstad
πŸ›
StuartClayton5
πŸ›
Supun Arunoda
πŸ›
Suren Abrahamyan
πŸ›
SwatiBGupta1110
πŸ›
SyedThoufich
πŸ›
Szymon Sasin
πŸ› -
T-chuangxin
πŸ› +
T-chuangxin
πŸ›
TERAI Atsuhiro
πŸ›
TIOBE Software
πŸ’» πŸ›
Taylor Smock
πŸ›
Techeira DamiΓ‘n
πŸ’» πŸ›
Ted Husted
πŸ›
TehBakker
πŸ› -
The Gitter Badger
πŸ› +
The Gitter Badger
πŸ›
Theodoor
πŸ›
Thiago Henrique HΓΌpner
πŸ›
Thibault Meyer
πŸ›
Thomas GΓΌttler
πŸ›
Thomas Jones-Low
πŸ›
Thomas Smith
πŸ’» πŸ› -
ThrawnCA
πŸ› +
ThrawnCA
πŸ›
Thunderforge
πŸ’» πŸ›
Tim van der Lippe
πŸ›
Tobias Weimer
πŸ’» πŸ›
Tom Daly
πŸ›
Tomer Figenblat
πŸ›
Tomi De Lucca
πŸ’» πŸ› -
Torsten Kleiber
πŸ› +
Torsten Kleiber
πŸ›
TrackerSB
πŸ›
Ullrich Hafner
πŸ›
Utku Cuhadaroglu
πŸ’» πŸ›
Valentin Brandl
πŸ›
Valeria
πŸ›
Vasily Anisimov
πŸ› -
Vibhor Goyal
πŸ› +
Vibhor Goyal
πŸ›
Vickenty Fesunov
πŸ›
Victor NoΓ«l
πŸ›
Vincent Galloy
πŸ’»
Vincent HUYNH
πŸ›
Vincent Maurin
πŸ›
Vincent Privat
πŸ› -
Vishhwas
πŸ› +
Vishhwas
πŸ›
Vitaly
πŸ›
Vitaly Polonetsky
πŸ›
Vojtech Polivka
πŸ›
Vsevolod Zholobov
πŸ›
Vyom Yadav
πŸ’»
Wang Shidong
πŸ› -
Waqas Ahmed
πŸ› +
Waqas Ahmed
πŸ›
Wayne J. Earl
πŸ›
Wchenghui
πŸ›
Will Winder
πŸ›
William Brockhus
πŸ’» πŸ›
Wilson Kurniawan
πŸ›
Wim Deblauwe
πŸ› -
Woongsik Choi
πŸ› +
Woongsik Choi
πŸ›
XenoAmess
πŸ’» πŸ›
Yang
πŸ’»
YaroslavTER
πŸ›
Young Chan
πŸ’» πŸ›
YuJin Kim
πŸ›
Yuri Dolzhenko
πŸ› -
Yurii Dubinka
πŸ› +
Yurii Dubinka
πŸ›
Zoltan Farkas
πŸ›
Zustin
πŸ›
aaronhurst-google
πŸ›
alexmodis
πŸ›
andreoss
πŸ›
andrey81inmd
πŸ’» πŸ› -
anicoara
πŸ› +
anicoara
πŸ›
arunprasathav
πŸ›
asiercamara
πŸ›
astillich-igniti
πŸ’»
avesolovksyy
πŸ›
avishvat
πŸ›
avivmu
πŸ› -
axelbarfod1
πŸ› +
axelbarfod1
πŸ›
b-3-n
πŸ›
balbhadra9
πŸ›
base23de
πŸ›
bergander
πŸ›
berkam
πŸ’» πŸ›
breizh31
πŸ› -
caesarkim
πŸ› +
caesarkim
πŸ›
carolyujing
πŸ›
cesares-basilico
πŸ›
chrite
πŸ›
cobratbq
πŸ›
coladict
πŸ›
cosmoJFH
πŸ› -
cristalp
πŸ› +
cristalp
πŸ›
crunsk
πŸ›
cwholmes
πŸ›
cyberjj999
πŸ›
cyw3
πŸ›
d1ss0nanz
πŸ›
dalizi007
πŸ’» -
danbrycefairsailcom
πŸ› +
danbrycefairsailcom
πŸ›
dariansanity
πŸ›
darrenmiliband
πŸ›
davidburstrom
πŸ›
dbirkman-paloalto
πŸ›
deepak-patra
πŸ›
dependabot[bot]
πŸ’» πŸ› -
dinesh150
πŸ› +
dinesh150
πŸ›
diziaq
πŸ›
dreaminpast123
πŸ›
duanyanan
πŸ›
dutt-sanjay
πŸ›
dylanleung
πŸ›
dzeigler
πŸ› -
ekkirala
πŸ› +
ekkirala
πŸ›
emersonmoura
πŸ›
fairy
πŸ›
filiprafalowicz
πŸ’»
foxmason
πŸ›
frankegabor
πŸ›
frankl
πŸ› -
freafrea
πŸ› +
freafrea
πŸ›
fsapatin
πŸ›
gracia19
πŸ›
guo fei
πŸ›
gurmsc5
πŸ›
gwilymatgearset
πŸ’» πŸ›
haigsn
πŸ› -
hemanshu070
πŸ› +
hemanshu070
πŸ›
henrik242
πŸ›
hongpuwu
πŸ›
hvbtup
πŸ’» πŸ›
igniti GmbH
πŸ›
ilovezfs
πŸ›
itaigilo
πŸ› -
jakivey32
πŸ› +
jakivey32
πŸ›
jbennett2091
πŸ›
jcamerin
πŸ›
jkeener1
πŸ›
jmetertea
πŸ›
johnra2
πŸ’»
josemanuelrolon
πŸ’» πŸ› -
kabroxiko
πŸ’» πŸ› +
kabroxiko
πŸ’» πŸ›
karwer
πŸ›
kaulonline
πŸ›
kdaemonv
πŸ›
kenji21
πŸ’» πŸ›
kfranic
πŸ›
khalidkh
πŸ› -
krzyk
πŸ› +
krzyk
πŸ›
lasselindqvist
πŸ›
lgemeinhardt
πŸ›
lihuaib
πŸ›
lonelyma1021
πŸ›
lpeddy
πŸ›
lujiefsi
πŸ’» -
lukelukes
πŸ’» +
lukelukes
πŸ’»
lyriccoder
πŸ›
marcelmore
πŸ›
matchbox
πŸ›
matthiaskraaz
πŸ›
meandonlyme
πŸ›
mikesive
πŸ› -
milossesic
πŸ› +
milossesic
πŸ›
mriddell95
πŸ›
mrlzh
πŸ›
msloan
πŸ›
mucharlaravalika
πŸ›
mvenneman
πŸ›
nareshl119
πŸ› -
nicolas-harraudeau-sonarsource
πŸ› +
nicolas-harraudeau-sonarsource
πŸ›
noerremark
πŸ›
novsirion
πŸ›
oggboy
πŸ›
oinume
πŸ›
orimarko
πŸ’» πŸ›
pallavi agarwal
πŸ› -
parksungrin
πŸ› +
parksungrin
πŸ›
patpatpat123
πŸ›
patriksevallius
πŸ›
pbrajesh1
πŸ›
phoenix384
πŸ›
piotrszymanski-sc
πŸ’»
plan3d
πŸ› -
poojasix
πŸ› +
poojasix
πŸ›
prabhushrikant
πŸ›
pujitha8783
πŸ›
r-r-a-j
πŸ›
raghujayjunk
πŸ›
rajeshveera
πŸ›
rajeswarreddy88
πŸ› -
recdevs
πŸ› +
recdevs
πŸ›
reudismam
πŸ’» πŸ›
rijkt
πŸ›
rillig-tk
πŸ›
rmohan20
πŸ’» πŸ›
rxmicro
πŸ›
ryan-gustafson
πŸ’» πŸ› -
sabi0
πŸ› +
sabi0
πŸ›
scais
πŸ›
sebbASF
πŸ›
sergeygorbaty
πŸ’»
shilko2013
πŸ›
shiomiyan
πŸ“–
simeonKondr
πŸ› -
snajberk
πŸ› +
snajberk
πŸ›
sniperrifle2004
πŸ›
snuyanzin
πŸ› πŸ’»
sratz
πŸ›
stonio
πŸ›
sturton
πŸ’» πŸ›
sudharmohan
πŸ› -
suruchidawar
πŸ› +
suruchidawar
πŸ›
svenfinitiv
πŸ›
tashiscool
πŸ›
test-git-hook
πŸ›
testation21
πŸ’» πŸ›
thanosa
πŸ›
tiandiyixian
πŸ› -
tobwoerk
πŸ› +
tobwoerk
πŸ›
tprouvot
πŸ›
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ›
tsui
πŸ›
winhkey
πŸ› -
witherspore
πŸ› +
witherspore
πŸ›
wjljack
πŸ›
wuchiuwong
πŸ›
xingsong
πŸ›
xioayuge
πŸ›
xnYi9wRezm
πŸ’» πŸ›
xuanuy
πŸ› -
xyf0921
πŸ› +
xyf0921
πŸ›
yalechen-cyw3
πŸ›
yasuharu-sato
πŸ›
zenglian
πŸ›
zgrzyt93
πŸ’» πŸ›
zh3ng
πŸ›
zt_soft
πŸ› -
ztt79
πŸ› +
ztt79
πŸ›
zzzzfeng
πŸ›
ÁrpÑd MagosÑnyi
πŸ›
任贡杰
πŸ› From f83a1a092cef9f09dbb7a5b3389a326ea89c21e7 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 17:55:04 +0200 Subject: [PATCH 102/136] [doc] Add deprecation infos --- docs/pages/release_notes.md | 15 +++++++++++++++ .../lang/apex/ast/ASTAssignmentExpression.java | 3 +++ .../pmd/lang/apex/ast/ASTBinaryExpression.java | 3 +++ .../pmd/lang/apex/ast/ASTBooleanExpression.java | 3 +++ .../pmd/lang/apex/ast/ASTPostfixExpression.java | 3 +++ .../pmd/lang/apex/ast/ASTPrefixExpression.java | 3 +++ 6 files changed, 30 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index f14ef9960d..53e43a3dad 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -21,6 +21,21 @@ This is a {{ site.pmd.release_type }} release. ### API Changes +#### Deprecated API + +* In order to reduce the dependency on Apex Jorje classes, the following methods have been deprecated. + These methods all leaked internal Jorje enums. These enums have been replaced now by enums the + PMD's AST package. + * {% jdoc !!apex::lang.apex.ast.ASTAssignmentExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTBinaryExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTBooleanExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTPostfixExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTPrefixExpression#getOperator() %} + + All these classes have now a new `getOp()` method. Existing code should be refactored to use this method instead. + It returns the new enums, like {% jdoc apex::lang.apex.ast.AssignmentOperator %}, and avoids + the dependency to Jorje. + ### External Contributions * [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java index 73b3b10740..b9e5c03027 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTAssignmentExpression.java @@ -22,6 +22,9 @@ public class ASTAssignmentExpression extends AbstractApexNode { return visitor.visit(this, data); } + /** + * @deprecated Use {@link #getOp()} instead. + */ @Deprecated public BinaryOp getOperator() { return node.getOp(); diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java index f258a389c8..04950e8907 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTBooleanExpression.java @@ -24,6 +24,9 @@ public class ASTBooleanExpression extends AbstractApexNode { return visitor.visit(this, data); } + /** + * @deprecated Use {@link #getOp()} instead. + */ @Deprecated public BooleanOp getOperator() { return this.node.getOp(); diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java index 8ef0b8770b..f3c9f858f7 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPostfixExpression.java @@ -24,6 +24,9 @@ public class ASTPostfixExpression extends AbstractApexNode { return visitor.visit(this, data); } + /** + * @deprecated Use {@link #getOp()} instead. + */ @Deprecated public PostfixOp getOperator() { return node.getOp(); diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java index bc8c4fdf29..049ac6b2b6 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTPrefixExpression.java @@ -22,6 +22,9 @@ public class ASTPrefixExpression extends AbstractApexNode { return visitor.visit(this, data); } + /** + * @deprecated Use {@link #getOp()} instead. + */ @Deprecated public PrefixOp getOperator() { return node.getOp(); From d2bd69437a761641bcb5459d741211d3c82f31eb Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 17:56:57 +0200 Subject: [PATCH 103/136] [doc] Update release notes (#4081) --- docs/pages/release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 53e43a3dad..5e50bcf8c9 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -38,6 +38,7 @@ This is a {{ site.pmd.release_type }} release. ### External Contributions +* [#4081](https://github.com/pmd/pmd/pull/4081): \[apex] Remove Jorje leaks outside `ast` package - [@eklimo](https://github.com/eklimo) * [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) {% endtocmaker %} From f71d68067c9da7f2edf73efd34c41be5e8820ece Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 17:57:23 +0200 Subject: [PATCH 104/136] Add @eklimo as a contributor --- .all-contributorsrc | 10 ++ docs/pages/pmd/projectdocs/credits.md | 163 +++++++++++++------------- 2 files changed, 92 insertions(+), 81 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 9ef6610038..78311a6858 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -6777,6 +6777,16 @@ "contributions": [ "code" ] + }, + { + "login": "eklimo", + "name": "Edward Klimoshenko", + "avatar_url": "https://avatars.githubusercontent.com/u/39220927?v=4", + "profile": "https://github.com/eklimo", + "contributions": [ + "bug", + "code" + ] } ], "contributorsPerLine": 7, diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 73955ce6f7..967675f0d1 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -231,734 +231,735 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Eden Hao
πŸ› +
Edward Klimoshenko
πŸ› πŸ’»
Egor Bredikhin
πŸ›
Elan P. Kugelmass
πŸ›
Elder S.
πŸ›
Emile
πŸ›
Eric
πŸ› -
Eric Kintzer
πŸ› +
Eric Kintzer
πŸ›
Eric Perret
πŸ›
Eric Squires
πŸ›
Erich L Foster
πŸ›
Erik Bleske
πŸ›
Ernst Reissner
πŸ›
F.W. Dekker
πŸ› -
FSchliephacke
πŸ› +
FSchliephacke
πŸ›
Facundo
πŸ›
Federico Giust
πŸ›
Fedor Sherstobitov
πŸ›
Felix Lampe
πŸ›
Filip Golonka
πŸ›
Filipe Esperandio
πŸ’» πŸ› -
Filippo Nova
πŸ› +
Filippo Nova
πŸ›
Francesco la Torre
πŸ›
Francisco Duarte
πŸ›
Frieder Bluemle
πŸ›
Frits Jalvingh
πŸ’» πŸ›
G. Bazior
πŸ›
Gabe Henkes
πŸ› -
Genoud Magloire
πŸ› +
Genoud Magloire
πŸ›
Geoffrey555
πŸ›
Georg Romstorfer
πŸ›
Gio
πŸ›
Gol
πŸ›
Gonzalo Exequiel Ibars Ingman
πŸ’» πŸ›
GooDer
πŸ› -
Gregor Riegler
πŸ› +
Gregor Riegler
πŸ›
Grzegorz Olszewski
πŸ›
Gunther Schrijvers
πŸ’» πŸ›
Gustavo Krieger
πŸ›
Guy Elsmore-Paddock
πŸ›
GΓΆrkem MΓΌlayim
πŸ›
Hanzel Godinez
πŸ› -
Haoliang Chen
πŸ› +
Haoliang Chen
πŸ›
Harsh Kukreja
πŸ›
Heber
πŸ›
Henning Schmiedehausen
πŸ’» πŸ›
Henning von Bargen
πŸ’»
HervΓ© Boutemy
πŸ›
Himanshu Pandey
πŸ› -
Hokwang Lee
πŸ› +
Hokwang Lee
πŸ›
Hooperbloob
πŸ’»
Hung PHAN
πŸ›
IDoCodingStuffs
πŸ’» πŸ›
Iccen Gan
πŸ›
Ignacio Mariano Tirabasso
πŸ›
Igor Melnichenko
πŸ› -
Igor Moreno
πŸ› +
Igor Moreno
πŸ›
Intelesis-MS
πŸ›
Iroha_
πŸ›
Ishan Srivastava
πŸ›
Ivano Guerini
πŸ›
Ivar Andreas Bonsaksen
πŸ›
Ivo Ε mΓ­d
πŸ› -
JJengility
πŸ› +
JJengility
πŸ›
Jake Hemmerle
πŸ›
James Harrison
πŸ› πŸ’»
Jan
πŸ›
Jan Aertgeerts
πŸ’» πŸ›
Jan BrΓΌmmer
πŸ›
Jan TΕ™Γ­ska
πŸ› -
Jan-Lukas Else
πŸ› +
Jan-Lukas Else
πŸ›
Jason Qiu
πŸ’» πŸ“–
Jason Williams
πŸ›
Jean-Paul Mayer
πŸ›
Jean-Simon Larochelle
πŸ›
Jeff Bartolotta
πŸ’» πŸ›
Jeff Hube
πŸ’» πŸ› -
Jeff Jensen
πŸ› +
Jeff Jensen
πŸ›
Jeff May
πŸ›
Jens Gerdes
πŸ›
Jeroen Borgers
πŸ› πŸ’»
Jerome Russ
πŸ›
JerritEic
πŸ’» πŸ“–
Jiri Pejchal
πŸ› -
Jithin Sunny
πŸ› +
Jithin Sunny
πŸ›
JiΕ™Γ­ Ε korpil
πŸ›
Joao Machado
πŸ›
Jochen Krauss
πŸ›
Johan Hammar
πŸ›
John Karp
πŸ›
John Zhang
πŸ› -
John-Teng
πŸ’» πŸ› +
John-Teng
πŸ’» πŸ›
Jon Moroney
πŸ’» πŸ›
Jonas Geiregat
πŸ›
Jonathan Wiesel
πŸ’» πŸ›
Jordan
πŸ›
Jordi Llach
πŸ›
Jorge SolΓ³rzano
πŸ› -
JorneVL
πŸ› +
JorneVL
πŸ›
Jose Palafox
πŸ›
Jose Stovall
πŸ›
Joseph
πŸ’»
Joseph Heenan
πŸ›
Josh Feingold
πŸ’» πŸ›
Josh Holthaus
πŸ› -
Joshua S Arquilevich
πŸ› +
Joshua S Arquilevich
πŸ›
JoΓ£o Ferreira
πŸ’» πŸ›
JoΓ£o Pedro Schmitt
πŸ›
Juan MartΓ­n Sotuyo Dodero
πŸ’» πŸ“– πŸ› 🚧
Juan Pablo Civile
πŸ›
Julian Voronetsky
πŸ›
Julien
πŸ› -
Julius
πŸ› +
Julius
πŸ›
JustPRV
πŸ›
JΓΆrn Huxhorn
πŸ›
KThompso
πŸ›
Kai Amundsen
πŸ›
Karel Vervaeke
πŸ›
Karl-Andero Mere
πŸ› -
Karl-Philipp Richter
πŸ› +
Karl-Philipp Richter
πŸ›
Karsten Silz
πŸ›
Kazuma Watanabe
πŸ›
Kev
πŸ›
Keve MΓΌller
πŸ›
Kevin Guerra
πŸ’»
Kevin Jones
πŸ› -
Kevin Wayne
πŸ› +
Kevin Wayne
πŸ›
Kieran Black
πŸ›
Kirill Zubov
πŸ›
Kirk Clemens
πŸ’» πŸ›
Klaus Hartl
πŸ›
Koen Van Looveren
πŸ›
Kris Scheibe
πŸ’» πŸ› -
Kunal Thanki
πŸ› +
Kunal Thanki
πŸ›
LaLucid
πŸ’»
Larry Diamond
πŸ’» πŸ›
Lars Knickrehm
πŸ›
Leo Gutierrez
πŸ›
LiGaOg
πŸ’»
Lintsi
πŸ› -
Linus Fernandes
πŸ› +
Linus Fernandes
πŸ›
Lixon Lookose
πŸ›
Logesh
πŸ›
Lorenzo Gabriele
πŸ›
LoΓ―c Ledoyen
πŸ›
Lucas Silva
πŸ›
Lucas Soncini
πŸ’» πŸ› -
Lukasz Slonina
πŸ› +
Lukasz Slonina
πŸ›
Lukebray
πŸ›
Lyor Goldstein
πŸ›
MCMicS
πŸ›
Macarse
πŸ›
Machine account for PMD
πŸ’»
Maciek Siemczyk
πŸ› -
Maikel Steneker
πŸ’» πŸ› +
Maikel Steneker
πŸ’» πŸ›
Maksim Moiseikin
πŸ›
Manfred Koch
πŸ›
Manuel Moya Ferrer
πŸ’» πŸ›
Manuel Ryan
πŸ›
Marat Vyshegorodtsev
πŸ›
Marcel HΓ€rle
πŸ› -
Marcello Fialho
πŸ› +
Marcello Fialho
πŸ›
Marcin Rataj
πŸ›
Mark Adamcin
πŸ›
Mark Hall
πŸ’» πŸ›
Mark Kolich
πŸ›
Mark Pritchard
πŸ›
Markus Rathgeb
πŸ› -
Marquis Wang
πŸ› +
Marquis Wang
πŸ›
Martin Feldsztejn
πŸ›
Martin Lehmann
πŸ›
Martin Spamer
πŸ›
Martin TarjΓ‘nyi
πŸ›
MatFl
πŸ›
Mateusz Stefanski
πŸ› -
Mathieu Gouin
πŸ› +
Mathieu Gouin
πŸ›
MatiasComercio
πŸ’» πŸ›
Matt Benson
πŸ›
Matt De Poorter
πŸ›
Matt Hargett
πŸ’» πŸ’΅
Matt Harrah
πŸ›
Matt Nelson
πŸ› -
Matthew Amos
πŸ› +
Matthew Amos
πŸ›
Matthew Duggan
πŸ›
Matthew Hall
πŸ›
MatΓ­as Fraga
πŸ’» πŸ›
Maxime Robert
πŸ’» πŸ›
MetaBF
πŸ›
Michael
πŸ› -
Michael Bell
πŸ› +
Michael Bell
πŸ›
Michael Bernstein
πŸ›
Michael Clay
πŸ›
Michael Dombrowski
πŸ›
Michael Hausegger
πŸ›
Michael Hoefer
πŸ›
Michael MΓΆbius
πŸ› -
Michael N. Lipp
πŸ› +
Michael N. Lipp
πŸ›
Michael Pellegrini
πŸ›
Michal Kordas
πŸ›
MichaΕ‚ Borek
πŸ›
MichaΕ‚ KuliΕ„ski
πŸ›
Miguel NΓΊΓ±ez DΓ­az-Montes
πŸ›
Mihai Ionut
πŸ› -
Mirek Hankus
πŸ› +
Mirek Hankus
πŸ›
Mladjan Gadzic
πŸ›
MrAngry52
πŸ›
Muminur Choudhury
πŸ›
Mykhailo Palahuta
πŸ’» πŸ›
Nagendra Kumar Singh
πŸ›
Nahuel Barrios
πŸ› -
Nathan Braun
πŸ› +
Nathan Braun
πŸ›
Nathan Reynolds
πŸ›
Nathan Reynolds
πŸ›
NathanaΓ«l
πŸ›
Naveen
πŸ’»
Nazdravi
πŸ›
Neha-Dhonde
πŸ› -
Nicholas Doyle
πŸ› +
Nicholas Doyle
πŸ›
Nick Butcher
πŸ›
Nico Gallinal
πŸ›
Nicola Dal Maso
πŸ›
Nicolas Filotto
πŸ’»
Nikita Chursin
πŸ›
Niklas Baudy
πŸ› -
Nikolas Havrikov
πŸ› +
Nikolas Havrikov
πŸ›
Nilesh Virkar
πŸ›
Nimit Patel
πŸ›
Niranjan Harpale
πŸ›
Noah Sussman
πŸ›
Noah0120
πŸ›
Noam Tamim
πŸ› -
Noel Grandin
πŸ› +
Noel Grandin
πŸ›
Olaf Haalstra
πŸ›
Oleg Pavlenko
πŸ›
Oleksii Dykov
πŸ’»
Oliver Eikemeier
πŸ›
Olivier Parent
πŸ’» πŸ›
Ollie Abbey
πŸ’» πŸ› -
OverDrone
πŸ› +
OverDrone
πŸ›
Ozan Gulle
πŸ’» πŸ›
PUNEET JAIN
πŸ›
Parbati Bose
πŸ›
Paul Berg
πŸ›
Pavel Bludov
πŸ›
Pavel Mička
πŸ› -
Pedro Nuno Santos
πŸ› +
Pedro Nuno Santos
πŸ›
Pedro Rijo
πŸ›
Pelisse Romain
πŸ’» πŸ“– πŸ›
Per Abich
πŸ’»
Pete Davids
πŸ›
Peter Bruin
πŸ›
Peter Chittum
πŸ’» πŸ› -
Peter Cudmore
πŸ› +
Peter Cudmore
πŸ›
Peter Kasson
πŸ›
Peter Kofler
πŸ›
Peter Paul Bakker
πŸ’»
Pham Hai Trung
πŸ›
Philip Graf
πŸ’» πŸ›
Philip Hachey
πŸ› -
Philippe Ozil
πŸ› +
Philippe Ozil
πŸ›
Phinehas Artemix
πŸ›
Phokham Nonava
πŸ›
Piotr SzymaΕ„ski
πŸ›
Piotrek Ε»ygieΕ‚o
πŸ’» πŸ›
Pranay Jaiswal
πŸ›
Prasad Kamath
πŸ› -
Prasanna
πŸ› +
Prasanna
πŸ›
Presh-AR
πŸ›
Puneet1726
πŸ›
Rafael CortΓͺs
πŸ›
RaheemShaik999
πŸ›
RajeshR
πŸ’» πŸ›
Ramachandra Mohan
πŸ› -
Ramel0921
πŸ› +
Ramel0921
πŸ›
Raquel Pau
πŸ›
Ravikiran Janardhana
πŸ›
Reda Benhemmouche
πŸ›
Renato Oliveira
πŸ’» πŸ›
Rich DiCroce
πŸ›
Riot R1cket
πŸ› -
Rishabh Jain
πŸ› +
Rishabh Jain
πŸ›
RishabhDeep Singh
πŸ›
Robbie Martinus
πŸ’» πŸ›
Robert Henry
πŸ›
Robert Painsi
πŸ›
Robert Russell
πŸ›
Robert SΓΆsemann
πŸ’» πŸ“– πŸ“’ πŸ› -
Robert Whitebit
πŸ› +
Robert Whitebit
πŸ›
Robin Richtsfeld
πŸ›
Robin Stocker
πŸ’» πŸ›
Robin Wils
πŸ›
RochusOest
πŸ›
Rodolfo Noviski
πŸ›
Rodrigo Casara
πŸ› -
Rodrigo Fernandes
πŸ› +
Rodrigo Fernandes
πŸ›
Roman Salvador
πŸ’» πŸ›
Ronald Blaschke
πŸ›
RΓ³bert Papp
πŸ›
Saikat Sengupta
πŸ›
Saksham Handu
πŸ›
Saladoc
πŸ› -
Salesforce Bob Lightning
πŸ› +
Salesforce Bob Lightning
πŸ›
Sam Carlberg
πŸ›
Satoshi Kubo
πŸ›
Scott Kennedy
πŸ›
Scott Wells
πŸ› πŸ’»
Scrsloota
πŸ’»
Sebastian BΓΆgl
πŸ› -
Sebastian Schuberth
πŸ› +
Sebastian Schuberth
πŸ›
Sebastian Schwarz
πŸ›
Sergey Gorbaty
πŸ›
Sergey Kozlov
πŸ›
Sergey Yanzin
πŸ’» πŸ›
Seth Wilcox
πŸ’»
Shubham
πŸ’» πŸ› -
Simon Abykov
πŸ’» +
Simon Abykov
πŸ’»
Simon Xiao
πŸ›
Srinivasan Venkatachalam
πŸ›
Stanislav Gromov
πŸ›
Stanislav Myachenkov
πŸ’»
Stefan Birkner
πŸ›
Stefan Bohn
πŸ› -
Stefan Endrullis
πŸ› +
Stefan Endrullis
πŸ›
Stefan KlΓΆss-Schuster
πŸ›
Stefan Wolf
πŸ›
Stephan H. Wissel
πŸ›
Stephen
πŸ›
Stephen Friedrich
πŸ›
Steve Babula
πŸ’» -
Stexxe
πŸ› +
Stexxe
πŸ›
Stian LΓ₯gstad
πŸ›
StuartClayton5
πŸ›
Supun Arunoda
πŸ›
Suren Abrahamyan
πŸ›
SwatiBGupta1110
πŸ›
SyedThoufich
πŸ› -
Szymon Sasin
πŸ› +
Szymon Sasin
πŸ›
T-chuangxin
πŸ›
TERAI Atsuhiro
πŸ›
TIOBE Software
πŸ’» πŸ›
Taylor Smock
πŸ›
Techeira DamiΓ‘n
πŸ’» πŸ›
Ted Husted
πŸ› -
TehBakker
πŸ› +
TehBakker
πŸ›
The Gitter Badger
πŸ›
Theodoor
πŸ›
Thiago Henrique HΓΌpner
πŸ›
Thibault Meyer
πŸ›
Thomas GΓΌttler
πŸ›
Thomas Jones-Low
πŸ› -
Thomas Smith
πŸ’» πŸ› +
Thomas Smith
πŸ’» πŸ›
ThrawnCA
πŸ›
Thunderforge
πŸ’» πŸ›
Tim van der Lippe
πŸ›
Tobias Weimer
πŸ’» πŸ›
Tom Daly
πŸ›
Tomer Figenblat
πŸ› -
Tomi De Lucca
πŸ’» πŸ› +
Tomi De Lucca
πŸ’» πŸ›
Torsten Kleiber
πŸ›
TrackerSB
πŸ›
Ullrich Hafner
πŸ›
Utku Cuhadaroglu
πŸ’» πŸ›
Valentin Brandl
πŸ›
Valeria
πŸ› -
Vasily Anisimov
πŸ› +
Vasily Anisimov
πŸ›
Vibhor Goyal
πŸ›
Vickenty Fesunov
πŸ›
Victor NoΓ«l
πŸ›
Vincent Galloy
πŸ’»
Vincent HUYNH
πŸ›
Vincent Maurin
πŸ› -
Vincent Privat
πŸ› +
Vincent Privat
πŸ›
Vishhwas
πŸ›
Vitaly
πŸ›
Vitaly Polonetsky
πŸ›
Vojtech Polivka
πŸ›
Vsevolod Zholobov
πŸ›
Vyom Yadav
πŸ’» -
Wang Shidong
πŸ› +
Wang Shidong
πŸ›
Waqas Ahmed
πŸ›
Wayne J. Earl
πŸ›
Wchenghui
πŸ›
Will Winder
πŸ›
William Brockhus
πŸ’» πŸ›
Wilson Kurniawan
πŸ› -
Wim Deblauwe
πŸ› +
Wim Deblauwe
πŸ›
Woongsik Choi
πŸ›
XenoAmess
πŸ’» πŸ›
Yang
πŸ’»
YaroslavTER
πŸ›
Young Chan
πŸ’» πŸ›
YuJin Kim
πŸ› -
Yuri Dolzhenko
πŸ› +
Yuri Dolzhenko
πŸ›
Yurii Dubinka
πŸ›
Zoltan Farkas
πŸ›
Zustin
πŸ›
aaronhurst-google
πŸ›
alexmodis
πŸ›
andreoss
πŸ› -
andrey81inmd
πŸ’» πŸ› +
andrey81inmd
πŸ’» πŸ›
anicoara
πŸ›
arunprasathav
πŸ›
asiercamara
πŸ›
astillich-igniti
πŸ’»
avesolovksyy
πŸ›
avishvat
πŸ› -
avivmu
πŸ› +
avivmu
πŸ›
axelbarfod1
πŸ›
b-3-n
πŸ›
balbhadra9
πŸ›
base23de
πŸ›
bergander
πŸ›
berkam
πŸ’» πŸ› -
breizh31
πŸ› +
breizh31
πŸ›
caesarkim
πŸ›
carolyujing
πŸ›
cesares-basilico
πŸ›
chrite
πŸ›
cobratbq
πŸ›
coladict
πŸ› -
cosmoJFH
πŸ› +
cosmoJFH
πŸ›
cristalp
πŸ›
crunsk
πŸ›
cwholmes
πŸ›
cyberjj999
πŸ›
cyw3
πŸ›
d1ss0nanz
πŸ› -
dalizi007
πŸ’» +
dalizi007
πŸ’»
danbrycefairsailcom
πŸ›
dariansanity
πŸ›
darrenmiliband
πŸ›
davidburstrom
πŸ›
dbirkman-paloalto
πŸ›
deepak-patra
πŸ› -
dependabot[bot]
πŸ’» πŸ› +
dependabot[bot]
πŸ’» πŸ›
dinesh150
πŸ›
diziaq
πŸ›
dreaminpast123
πŸ›
duanyanan
πŸ›
dutt-sanjay
πŸ›
dylanleung
πŸ› -
dzeigler
πŸ› +
dzeigler
πŸ›
ekkirala
πŸ›
emersonmoura
πŸ›
fairy
πŸ›
filiprafalowicz
πŸ’»
foxmason
πŸ›
frankegabor
πŸ› -
frankl
πŸ› +
frankl
πŸ›
freafrea
πŸ›
fsapatin
πŸ›
gracia19
πŸ›
guo fei
πŸ›
gurmsc5
πŸ›
gwilymatgearset
πŸ’» πŸ› -
haigsn
πŸ› +
haigsn
πŸ›
hemanshu070
πŸ›
henrik242
πŸ›
hongpuwu
πŸ›
hvbtup
πŸ’» πŸ›
igniti GmbH
πŸ›
ilovezfs
πŸ› -
itaigilo
πŸ› +
itaigilo
πŸ›
jakivey32
πŸ›
jbennett2091
πŸ›
jcamerin
πŸ›
jkeener1
πŸ›
jmetertea
πŸ›
johnra2
πŸ’» -
josemanuelrolon
πŸ’» πŸ› +
josemanuelrolon
πŸ’» πŸ›
kabroxiko
πŸ’» πŸ›
karwer
πŸ›
kaulonline
πŸ›
kdaemonv
πŸ›
kenji21
πŸ’» πŸ›
kfranic
πŸ› -
khalidkh
πŸ› +
khalidkh
πŸ›
krzyk
πŸ›
lasselindqvist
πŸ›
lgemeinhardt
πŸ›
lihuaib
πŸ›
lonelyma1021
πŸ›
lpeddy
πŸ› -
lujiefsi
πŸ’» +
lujiefsi
πŸ’»
lukelukes
πŸ’»
lyriccoder
πŸ›
marcelmore
πŸ›
matchbox
πŸ›
matthiaskraaz
πŸ›
meandonlyme
πŸ› -
mikesive
πŸ› +
mikesive
πŸ›
milossesic
πŸ›
mriddell95
πŸ›
mrlzh
πŸ›
msloan
πŸ›
mucharlaravalika
πŸ›
mvenneman
πŸ› -
nareshl119
πŸ› +
nareshl119
πŸ›
nicolas-harraudeau-sonarsource
πŸ›
noerremark
πŸ›
novsirion
πŸ›
oggboy
πŸ›
oinume
πŸ›
orimarko
πŸ’» πŸ› -
pallavi agarwal
πŸ› +
pallavi agarwal
πŸ›
parksungrin
πŸ›
patpatpat123
πŸ›
patriksevallius
πŸ›
pbrajesh1
πŸ›
phoenix384
πŸ›
piotrszymanski-sc
πŸ’» -
plan3d
πŸ› +
plan3d
πŸ›
poojasix
πŸ›
prabhushrikant
πŸ›
pujitha8783
πŸ›
r-r-a-j
πŸ›
raghujayjunk
πŸ›
rajeshveera
πŸ› -
rajeswarreddy88
πŸ› +
rajeswarreddy88
πŸ›
recdevs
πŸ›
reudismam
πŸ’» πŸ›
rijkt
πŸ›
rillig-tk
πŸ›
rmohan20
πŸ’» πŸ›
rxmicro
πŸ› -
ryan-gustafson
πŸ’» πŸ› +
ryan-gustafson
πŸ’» πŸ›
sabi0
πŸ›
scais
πŸ›
sebbASF
πŸ›
sergeygorbaty
πŸ’»
shilko2013
πŸ›
shiomiyan
πŸ“– -
simeonKondr
πŸ› +
simeonKondr
πŸ›
snajberk
πŸ›
sniperrifle2004
πŸ›
snuyanzin
πŸ› πŸ’»
sratz
πŸ›
stonio
πŸ›
sturton
πŸ’» πŸ› -
sudharmohan
πŸ› +
sudharmohan
πŸ›
suruchidawar
πŸ›
svenfinitiv
πŸ›
tashiscool
πŸ›
test-git-hook
πŸ›
testation21
πŸ’» πŸ›
thanosa
πŸ› -
tiandiyixian
πŸ› +
tiandiyixian
πŸ›
tobwoerk
πŸ›
tprouvot
πŸ›
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ›
tsui
πŸ› -
winhkey
πŸ› +
winhkey
πŸ›
witherspore
πŸ›
wjljack
πŸ›
wuchiuwong
πŸ›
xingsong
πŸ›
xioayuge
πŸ›
xnYi9wRezm
πŸ’» πŸ› -
xuanuy
πŸ› +
xuanuy
πŸ›
xyf0921
πŸ›
yalechen-cyw3
πŸ›
yasuharu-sato
πŸ›
zenglian
πŸ›
zgrzyt93
πŸ’» πŸ›
zh3ng
πŸ› -
zt_soft
πŸ› +
zt_soft
πŸ›
ztt79
πŸ›
zzzzfeng
πŸ›
ÁrpÑd MagosÑnyi
πŸ› From 67bab1609179fbc4168a6202b3e86b3032234430 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 17:59:57 +0200 Subject: [PATCH 105/136] [doc] Update release notes (#4092) --- docs/pages/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..3cfcae03f1 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -20,5 +20,7 @@ This is a {{ site.pmd.release_type }} release. ### External Contributions +* [#4092](https://github.com/pmd/pmd/pull/4092): \[apex] Implement ApexQualifiableNode for ASTUserEnum - [@aaronhurst-google](https://github.com/aaronhurst-google) + {% endtocmaker %} From 4847470db0d898b6dcf622398b1710d8fd9343f7 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 18:00:17 +0200 Subject: [PATCH 106/136] Update @aaronhurst-google as a contributor --- .all-contributorsrc | 3 ++- docs/pages/pmd/projectdocs/credits.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 99708783e8..0c84c82931 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1053,7 +1053,8 @@ "avatar_url": "https://avatars.githubusercontent.com/u/86377278?v=4", "profile": "https://github.com/aaronhurst-google", "contributions": [ - "bug" + "bug", + "code" ] }, { diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 37788cf5f7..d8b9a5408b 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -736,7 +736,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Zoltan Farkas
πŸ›
Zustin
πŸ› -
aaronhurst-google
πŸ› +
aaronhurst-google
πŸ› πŸ’»
alexmodis
πŸ›
andreoss
πŸ›
andrey81inmd
πŸ’» πŸ› From 5d5fbd5b21ce3e016b7952676c2b7d79bda51a6b Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 19:15:56 +0200 Subject: [PATCH 107/136] Bump maven-pmd-plugin from 3.17.0 to 3.18.0 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 1f7aa9e9b7..5b215e903e 100644 --- a/pom.xml +++ b/pom.xml @@ -94,7 +94,7 @@ 3.0.0-M5 9.3 3.1.2 - 3.17.0 + 3.18.0 1.10.12 3.2.0 4.7.2 From 804e4d188afcffb2b430ea3d079ee0e12f555cca Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 24 Aug 2022 19:41:08 +0200 Subject: [PATCH 108/136] [ci] Allow building of branch "experimental-apex-parser" It should build like a pull request --- .ci/build.sh | 9 +++++++++ .github/workflows/build.yml | 1 + 2 files changed, 10 insertions(+) diff --git a/.ci/build.sh b/.ci/build.sh index 92f7f6472a..6c6b657654 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -27,6 +27,15 @@ function build() { pmd_ci_utils_determine_build_env pmd/pmd echo + if ! pmd_ci_utils_is_fork_or_pull_request; then + if [ "${PMD_CI_BRANCH}" = "experimental-apex-parser" ]; then + pmd_ci_log_group_start "Build with mvnw" + ./mvnw clean install --show-version --errors --batch-mode --no-transfer-progress "${PMD_MAVEN_EXTRA_OPTS[@]}" + pmd_ci_log_group_end + exit 0 + fi + fi + if pmd_ci_utils_is_fork_or_pull_request; then pmd_ci_log_group_start "Build with mvnw" ./mvnw clean install --show-version --errors --batch-mode --no-transfer-progress "${PMD_MAVEN_EXTRA_OPTS[@]}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1f64ba25cb..f94319eb05 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,6 +5,7 @@ on: branches: - main - master + - experimental-apex-parser tags: - '**' pull_request: From e0b9965c27f8663135d4be915b850184c58927e7 Mon Sep 17 00:00:00 2001 From: tprouvot <35368290+tprouvot@users.noreply.github.com> Date: Thu, 25 Aug 2022 10:46:47 +0200 Subject: [PATCH 109/136] Update pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java Co-authored-by: Andreas Dangel --- .../bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java index 2e0d717b60..71f2ea9b99 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java @@ -43,7 +43,7 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest ASSERT_METHODS.add("assert.isfalse"); ASSERT_METHODS.add("assert.isinstanceoftype"); ASSERT_METHODS.add("assert.isnotinstanceoftype"); - ASSERT_METHODS.add("assert.isnnull"); + ASSERT_METHODS.add("assert.isnull"); ASSERT_METHODS.add("assert.isnotnull"); ASSERT_METHODS.add("assert.istrue"); // Fully-qualified variants...rare but still valid/possible From d5098ca115de49e450be43415d238c37f0d13987 Mon Sep 17 00:00:00 2001 From: tprouvot <35368290+tprouvot@users.noreply.github.com> Date: Thu, 25 Aug 2022 12:15:30 +0200 Subject: [PATCH 110/136] Update pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml Co-authored-by: Andreas Dangel --- .../bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index b78a6c86f7..471822ad5a 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -117,7 +117,7 @@ public class Foo { - #4096 [apex] api 56.0 ApexAssertionsShouldIncludeMessage and new apex class : Assert + [apex] ApexUnitTestClassShouldHaveAssertsRule: Support new Assert class (Apex v56.0) #4097 0 Date: Thu, 25 Aug 2022 12:17:32 +0200 Subject: [PATCH 111/136] Fix typo --- ...pexUnitTestClassShouldHaveAssertsRule.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java index 2e0d717b60..d45f93c397 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexUnitTestClassShouldHaveAssertsRule.java @@ -56,17 +56,20 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest ASSERT_METHODS.add("system.assert.isfalse"); ASSERT_METHODS.add("system.assert.isinstanceoftype"); ASSERT_METHODS.add("system.assert.isnotinstanceoftype"); - ASSERT_METHODS.add("system.assert.isnnull"); + ASSERT_METHODS.add("system.assert.isnull"); ASSERT_METHODS.add("system.assert.isnotnull"); ASSERT_METHODS.add("system.assert.istrue"); } - // Using a string property instead of a regex property to ensure that the compiled pattern can be case-insensitive - private static final PropertyDescriptor ADDITIONAL_ASSERT_METHOD_PATTERN_DESCRIPTOR = - stringProperty("additionalAssertMethodPattern") - .desc("A regular expression for one or more custom test assertion method patterns.").defaultValue("").build(); + // Using a string property instead of a regex property to ensure that the + // compiled pattern can be case-insensitive + private static final PropertyDescriptor ADDITIONAL_ASSERT_METHOD_PATTERN_DESCRIPTOR = stringProperty( + "additionalAssertMethodPattern") + .desc("A regular expression for one or more custom test assertion method patterns.").defaultValue("") + .build(); - // A simple compiled pattern cache to ensure that we only ever try to compile the configured pattern once for a given run + // A simple compiled pattern cache to ensure that we only ever try to compile + // the configured pattern once for a given run private Optional compiledAdditionalAssertMethodPattern = null; public ApexUnitTestClassShouldHaveAssertsRule() { @@ -99,7 +102,8 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest } } - // If we didn't find assert method invocations the simple way and we have a configured pattern, try it + // If we didn't find assert method invocations the simple way and we have a + // configured pattern, try it if (!isAssertFound) { final String additionalAssertMethodPattern = getProperty(ADDITIONAL_ASSERT_METHOD_PATTERN_DESCRIPTOR); final Pattern compiledPattern = getCompiledAdditionalAssertMethodPattern(additionalAssertMethodPattern); @@ -123,12 +127,15 @@ public class ApexUnitTestClassShouldHaveAssertsRule extends AbstractApexUnitTest private Pattern getCompiledAdditionalAssertMethodPattern(String additionalAssertMethodPattern) { if (StringUtils.isNotBlank(additionalAssertMethodPattern)) { - // Check for presence first since we will cache a null value for patterns that don't compile + // Check for presence first since we will cache a null value for patterns that + // don't compile if (compiledAdditionalAssertMethodPattern == null) { try { - compiledAdditionalAssertMethodPattern = Optional.of(Pattern.compile(additionalAssertMethodPattern, Pattern.CASE_INSENSITIVE)); + compiledAdditionalAssertMethodPattern = Optional + .of(Pattern.compile(additionalAssertMethodPattern, Pattern.CASE_INSENSITIVE)); } catch (IllegalArgumentException e) { - // Cache a null compiled pattern so that we won't try to compile this one again during the run + // Cache a null compiled pattern so that we won't try to compile this one again + // during the run compiledAdditionalAssertMethodPattern = Optional.ofNullable(null); throw e; } From 2e9f25c8ba8ca963660dd604e81fe54fa625e508 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 25 Aug 2022 12:17:51 +0200 Subject: [PATCH 112/136] PR review comment --- .../bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index b78a6c86f7..f08128a6f7 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -102,8 +102,8 @@ public class Foo { Date: Thu, 25 Aug 2022 12:18:15 +0200 Subject: [PATCH 113/136] Add new Assert methods to ApexAssertionsShouldIncludeMessageRule --- ...pexAssertionsShouldIncludeMessageRule.java | 26 +++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java index 679935d617..fa50f75334 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java @@ -12,17 +12,39 @@ public class ApexAssertionsShouldIncludeMessageRule extends AbstractApexUnitTest private static final String ASSERT = "System.assert"; private static final String ASSERT_EQUALS = "System.assertEquals"; private static final String ASSERT_NOT_EQUALS = "System.assertNotEquals"; + private static final String ARE_EQUAL = "Assert.areEqual"; + private static final String ARE_NOT_EQUAL = "Assert.areNotEqual"; + private static final String IS_FALSE = "Assert.isFalse"; + private static final String FAIL = "Assert.fail"; + private static final String IS_INSTANCE_OF_TYPE = "Assert.isInstanceOfType"; + private static final String IS_NOT_INSTANCE_OF_TYPE = "Assert.isNotInstanceOfType"; + private static final String IS_NOT_NULL = "Assert.isNotNull"; + private static final String IS_NULL = "Assert.isNull"; + private static final String IS_TRUE = "Assert.isTrue"; @Override public Object visit(ASTMethodCallExpression node, Object data) { String methodName = node.getFullMethodName(); - if (ASSERT.equalsIgnoreCase(methodName) && node.getNumChildren() == 2) { + if (FAIL.equalsIgnoreCase(methodName) && node.getNumChildren() == 1) { + addViolationWithMessage(data, node, + "''{0}'' should have 1 parameters.", + new Object[] { FAIL }); + } else if ((ASSERT.equalsIgnoreCase(methodName) + || IS_FALSE.equalsIgnoreCase(methodName) + || IS_NOT_NULL.equalsIgnoreCase(methodName) + || IS_NULL.equalsIgnoreCase(methodName) + || IS_TRUE.equalsIgnoreCase(methodName)) + && node.getNumChildren() == 2) { addViolationWithMessage(data, node, "''{0}'' should have 2 parameters.", new Object[] { ASSERT }); } else if ((ASSERT_EQUALS.equalsIgnoreCase(methodName) - || ASSERT_NOT_EQUALS.equalsIgnoreCase(methodName)) + || ASSERT_NOT_EQUALS.equalsIgnoreCase(methodName) + || ARE_EQUAL.equalsIgnoreCase(methodName) + || ARE_NOT_EQUAL.equalsIgnoreCase(methodName) + || IS_INSTANCE_OF_TYPE.equalsIgnoreCase(methodName) + || IS_NOT_INSTANCE_OF_TYPE.equalsIgnoreCase(methodName)) && node.getNumChildren() == 3) { addViolationWithMessage(data, node, "''{0}'' should have 3 parameters.", From d6194be47b39cb7f41b665ceb08e5b4e25d82e95 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 25 Aug 2022 15:31:13 +0200 Subject: [PATCH 114/136] Add tests for new methods --- .../ApexUnitTestClassShouldHaveAsserts.xml | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index 048170d3ad..0ce6d81f0f 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -122,9 +122,53 @@ public class Foo { From ac94df9e17f6778967cb3e63516e9c0cf7260e80 Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 25 Aug 2022 15:34:54 +0200 Subject: [PATCH 115/136] Update add violation parameters to macth with new tests --- .../bestpractices/ApexAssertionsShouldIncludeMessageRule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java index fa50f75334..2919206121 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/rule/bestpractices/ApexAssertionsShouldIncludeMessageRule.java @@ -38,7 +38,7 @@ public class ApexAssertionsShouldIncludeMessageRule extends AbstractApexUnitTest && node.getNumChildren() == 2) { addViolationWithMessage(data, node, "''{0}'' should have 2 parameters.", - new Object[] { ASSERT }); + new Object[] { methodName }); } else if ((ASSERT_EQUALS.equalsIgnoreCase(methodName) || ASSERT_NOT_EQUALS.equalsIgnoreCase(methodName) || ARE_EQUAL.equalsIgnoreCase(methodName) From 065b4b80835c9d27d96516cabbbd3d761709a6ac Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 25 Aug 2022 15:44:18 +0200 Subject: [PATCH 116/136] Remove param message not mandatory in this test --- .../xml/ApexUnitTestClassShouldHaveAsserts.xml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index 0ce6d81f0f..1e16256b0a 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -124,18 +124,18 @@ public class Foo { public class Foo { public static testAreEqual void testSomething() { String sub = 'abcde'.substring(2); - Assert.areEqual('cde', sub, 'Expected characters after first two'); + Assert.areEqual('cde', sub); } public static testAreNotEqual void testSomething() { String sub = 'abcde'.substring(2); - Assert.areNotEqual('xyz', sub, 'Characters not expected after first two'); + Assert.areNotEqual('xyz', sub); } public static testFail void testSomething() { try { SomeClass.methodUnderTest(); - Assert.fail('DmlException Expected'); + Assert.fail(); } catch (DmlException ex) { // Add assertions here about the expected exception } @@ -143,7 +143,7 @@ public class Foo { public static testIsFalse void testSomething() { Boolean containsCode = 'Salesforce'.contains('code'); - Assert.isFalse(containsCode, 'No code'); + Assert.isFalse(containsCode); } public static testIsInstanceOf void testSomething() { @@ -153,22 +153,22 @@ public class Foo { public static testIsNotInstanceOf void testSomething() { Contact con = new Contact(); - Assert.isNotInstanceOfType(con, Account.class, 'Not expected type'); + Assert.isNotInstanceOfType(con, Account.class); } public static testIsNotNull void testSomething() { String myString = 'value'; - Assert.isNotNull(myString, 'myString should not be null'); + Assert.isNotNull(myString); } public static testIsNull void testSomething() { String myString = null; - Assert.isNull(myString, 'String should be null'); + Assert.isNull(myString); } public static testIsTrue void testSomething() { Boolean containsForce = 'Salesforce'.contains('force'); - Assert.isTrue(containsForce, 'Contains force'); + Assert.isTrue(containsForce); } } ]]> From d73cdcc193689fa31f524dabd0378ac0e55b324d Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 25 Aug 2022 15:44:37 +0200 Subject: [PATCH 117/136] Ad new tests for Assert method class --- .../ApexAssertionsShouldIncludeMessage.xml | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml index 71790b39db..ac608c7529 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml @@ -49,4 +49,62 @@ public class Foo { } ]]> + + + [apex] Support new Assert class (Apex v56.0) #4097 + 0 + + From 29c4e996fb0be080dea93df56bb4f9a438ad3e4e Mon Sep 17 00:00:00 2001 From: Thomas Prouvot Date: Thu, 25 Aug 2022 16:54:11 +0200 Subject: [PATCH 118/136] Increase test coverage --- .../ApexAssertionsShouldIncludeMessage.xml | 77 +++++++++++-------- .../ApexUnitTestClassShouldHaveAsserts.xml | 31 +++++--- 2 files changed, 63 insertions(+), 45 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml index ac608c7529..f05e9124da 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml @@ -50,61 +50,70 @@ public class Foo { ]]> - + [apex] Support new Assert class (Apex v56.0) #4097 0 - + ]]> diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index 1e16256b0a..28ba29690b 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -122,51 +122,60 @@ public class Foo { Date: Sun, 28 Aug 2022 15:00:55 +0200 Subject: [PATCH 119/136] Add MegaLinter in the list of integrations --- docs/pages/pmd/userdocs/tools/ci.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/pages/pmd/userdocs/tools/ci.md b/docs/pages/pmd/userdocs/tools/ci.md index f12b366935..513374f26e 100644 --- a/docs/pages/pmd/userdocs/tools/ci.md +++ b/docs/pages/pmd/userdocs/tools/ci.md @@ -28,4 +28,10 @@ result of the PMD maven plugin. See [Other Tools / Integrations](pmd_userdocs_tools.html#github-actions) +## MegaLinter + +[πŸ¦™ Mega-Linter](https://oxsecurity.github.io/megalinter/latest/) analyzes 50 languages, 22 formats, 21 tooling formats, excessive copy-pastes, spelling mistakes and security issues in your repository sources with a GitHub Action, other CI tools or locally. + +It [natively embeds PMD](https://oxsecurity.github.io/megalinter/latest/descriptors/java_pmd/). + From b83ca316e1863f5d3321dffa47afe07278a0f621 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Mon, 29 Aug 2022 18:33:17 +0200 Subject: [PATCH 120/136] [doc] Update release notes (#4104) --- docs/pages/release_notes.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 7fcbc0c973..8127c2c6a0 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -41,6 +41,7 @@ This is a {{ site.pmd.release_type }} release. * [#4081](https://github.com/pmd/pmd/pull/4081): \[apex] Remove Jorje leaks outside `ast` package - [@eklimo](https://github.com/eklimo) * [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) * [#4092](https://github.com/pmd/pmd/pull/4092): \[apex] Implement ApexQualifiableNode for ASTUserEnum - [@aaronhurst-google](https://github.com/aaronhurst-google) +* [#4104](https://github.com/pmd/pmd/pull/4104): \[doc] Add MegaLinter in the list of integrations - [@nvuillam](https://github.com/nvuillam) {% endtocmaker %} From bb81bd785710a94546f63b6bba3075bfb0c0bbe6 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Mon, 29 Aug 2022 18:34:52 +0200 Subject: [PATCH 121/136] Add @nvuillam as a contributor --- .all-contributorsrc | 9 +++ docs/pages/pmd/projectdocs/credits.md | 101 +++++++++++++------------- 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 87892e6d84..e87abaa8ae 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -6788,6 +6788,15 @@ "bug", "code" ] + }, + { + "login": "nvuillam", + "name": "Nicolas Vuillamy", + "avatar_url": "https://avatars.githubusercontent.com/u/17500430?v=4", + "profile": "https://github.com/nvuillam", + "contributions": [ + "doc" + ] } ], "contributorsPerLine": 7, diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index db79a432d7..ffb67b7978 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -514,451 +514,452 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
Nico Gallinal
πŸ›
Nicola Dal Maso
πŸ›
Nicolas Filotto
πŸ’» +
Nicolas Vuillamy
πŸ“–
Nikita Chursin
πŸ› -
Niklas Baudy
πŸ› +
Niklas Baudy
πŸ›
Nikolas Havrikov
πŸ›
Nilesh Virkar
πŸ›
Nimit Patel
πŸ›
Niranjan Harpale
πŸ›
Noah Sussman
πŸ›
Noah0120
πŸ› -
Noam Tamim
πŸ› +
Noam Tamim
πŸ›
Noel Grandin
πŸ›
Olaf Haalstra
πŸ›
Oleg Pavlenko
πŸ›
Oleksii Dykov
πŸ’»
Oliver Eikemeier
πŸ›
Olivier Parent
πŸ’» πŸ› -
Ollie Abbey
πŸ’» πŸ› +
Ollie Abbey
πŸ’» πŸ›
OverDrone
πŸ›
Ozan Gulle
πŸ’» πŸ›
PUNEET JAIN
πŸ›
Parbati Bose
πŸ›
Paul Berg
πŸ›
Pavel Bludov
πŸ› -
Pavel Mička
πŸ› +
Pavel Mička
πŸ›
Pedro Nuno Santos
πŸ›
Pedro Rijo
πŸ›
Pelisse Romain
πŸ’» πŸ“– πŸ›
Per Abich
πŸ’»
Pete Davids
πŸ›
Peter Bruin
πŸ› -
Peter Chittum
πŸ’» πŸ› +
Peter Chittum
πŸ’» πŸ›
Peter Cudmore
πŸ›
Peter Kasson
πŸ›
Peter Kofler
πŸ›
Peter Paul Bakker
πŸ’»
Pham Hai Trung
πŸ›
Philip Graf
πŸ’» πŸ› -
Philip Hachey
πŸ› +
Philip Hachey
πŸ›
Philippe Ozil
πŸ›
Phinehas Artemix
πŸ›
Phokham Nonava
πŸ›
Piotr SzymaΕ„ski
πŸ›
Piotrek Ε»ygieΕ‚o
πŸ’» πŸ›
Pranay Jaiswal
πŸ› -
Prasad Kamath
πŸ› +
Prasad Kamath
πŸ›
Prasanna
πŸ›
Presh-AR
πŸ›
Puneet1726
πŸ›
Rafael CortΓͺs
πŸ›
RaheemShaik999
πŸ›
RajeshR
πŸ’» πŸ› -
Ramachandra Mohan
πŸ› +
Ramachandra Mohan
πŸ›
Ramel0921
πŸ›
Raquel Pau
πŸ›
Ravikiran Janardhana
πŸ›
Reda Benhemmouche
πŸ›
Renato Oliveira
πŸ’» πŸ›
Rich DiCroce
πŸ› -
Riot R1cket
πŸ› +
Riot R1cket
πŸ›
Rishabh Jain
πŸ›
RishabhDeep Singh
πŸ›
Robbie Martinus
πŸ’» πŸ›
Robert Henry
πŸ›
Robert Painsi
πŸ›
Robert Russell
πŸ› -
Robert SΓΆsemann
πŸ’» πŸ“– πŸ“’ πŸ› +
Robert SΓΆsemann
πŸ’» πŸ“– πŸ“’ πŸ›
Robert Whitebit
πŸ›
Robin Richtsfeld
πŸ›
Robin Stocker
πŸ’» πŸ›
Robin Wils
πŸ›
RochusOest
πŸ›
Rodolfo Noviski
πŸ› -
Rodrigo Casara
πŸ› +
Rodrigo Casara
πŸ›
Rodrigo Fernandes
πŸ›
Roman Salvador
πŸ’» πŸ›
Ronald Blaschke
πŸ›
RΓ³bert Papp
πŸ›
Saikat Sengupta
πŸ›
Saksham Handu
πŸ› -
Saladoc
πŸ› +
Saladoc
πŸ›
Salesforce Bob Lightning
πŸ›
Sam Carlberg
πŸ›
Satoshi Kubo
πŸ›
Scott Kennedy
πŸ›
Scott Wells
πŸ› πŸ’»
Scrsloota
πŸ’» -
Sebastian BΓΆgl
πŸ› +
Sebastian BΓΆgl
πŸ›
Sebastian Schuberth
πŸ›
Sebastian Schwarz
πŸ›
Sergey Gorbaty
πŸ›
Sergey Kozlov
πŸ›
Sergey Yanzin
πŸ’» πŸ›
Seth Wilcox
πŸ’» -
Shubham
πŸ’» πŸ› +
Shubham
πŸ’» πŸ›
Simon Abykov
πŸ’»
Simon Xiao
πŸ›
Srinivasan Venkatachalam
πŸ›
Stanislav Gromov
πŸ›
Stanislav Myachenkov
πŸ’»
Stefan Birkner
πŸ› -
Stefan Bohn
πŸ› +
Stefan Bohn
πŸ›
Stefan Endrullis
πŸ›
Stefan KlΓΆss-Schuster
πŸ›
Stefan Wolf
πŸ›
Stephan H. Wissel
πŸ›
Stephen
πŸ›
Stephen Friedrich
πŸ› -
Steve Babula
πŸ’» +
Steve Babula
πŸ’»
Stexxe
πŸ›
Stian LΓ₯gstad
πŸ›
StuartClayton5
πŸ›
Supun Arunoda
πŸ›
Suren Abrahamyan
πŸ›
SwatiBGupta1110
πŸ› -
SyedThoufich
πŸ› +
SyedThoufich
πŸ›
Szymon Sasin
πŸ›
T-chuangxin
πŸ›
TERAI Atsuhiro
πŸ›
TIOBE Software
πŸ’» πŸ›
Taylor Smock
πŸ›
Techeira DamiΓ‘n
πŸ’» πŸ› -
Ted Husted
πŸ› +
Ted Husted
πŸ›
TehBakker
πŸ›
The Gitter Badger
πŸ›
Theodoor
πŸ›
Thiago Henrique HΓΌpner
πŸ›
Thibault Meyer
πŸ›
Thomas GΓΌttler
πŸ› -
Thomas Jones-Low
πŸ› +
Thomas Jones-Low
πŸ›
Thomas Smith
πŸ’» πŸ›
ThrawnCA
πŸ›
Thunderforge
πŸ’» πŸ›
Tim van der Lippe
πŸ›
Tobias Weimer
πŸ’» πŸ›
Tom Daly
πŸ› -
Tomer Figenblat
πŸ› +
Tomer Figenblat
πŸ›
Tomi De Lucca
πŸ’» πŸ›
Torsten Kleiber
πŸ›
TrackerSB
πŸ›
Ullrich Hafner
πŸ›
Utku Cuhadaroglu
πŸ’» πŸ›
Valentin Brandl
πŸ› -
Valeria
πŸ› +
Valeria
πŸ›
Vasily Anisimov
πŸ›
Vibhor Goyal
πŸ›
Vickenty Fesunov
πŸ›
Victor NoΓ«l
πŸ›
Vincent Galloy
πŸ’»
Vincent HUYNH
πŸ› -
Vincent Maurin
πŸ› +
Vincent Maurin
πŸ›
Vincent Privat
πŸ›
Vishhwas
πŸ›
Vitaly
πŸ›
Vitaly Polonetsky
πŸ›
Vojtech Polivka
πŸ›
Vsevolod Zholobov
πŸ› -
Vyom Yadav
πŸ’» +
Vyom Yadav
πŸ’»
Wang Shidong
πŸ›
Waqas Ahmed
πŸ›
Wayne J. Earl
πŸ›
Wchenghui
πŸ›
Will Winder
πŸ›
William Brockhus
πŸ’» πŸ› -
Wilson Kurniawan
πŸ› +
Wilson Kurniawan
πŸ›
Wim Deblauwe
πŸ›
Woongsik Choi
πŸ›
XenoAmess
πŸ’» πŸ›
Yang
πŸ’»
YaroslavTER
πŸ›
Young Chan
πŸ’» πŸ› -
YuJin Kim
πŸ› +
YuJin Kim
πŸ›
Yuri Dolzhenko
πŸ›
Yurii Dubinka
πŸ›
Zoltan Farkas
πŸ›
Zustin
πŸ›
aaronhurst-google
πŸ› πŸ’»
alexmodis
πŸ› -
andreoss
πŸ› +
andreoss
πŸ›
andrey81inmd
πŸ’» πŸ›
anicoara
πŸ›
arunprasathav
πŸ›
asiercamara
πŸ›
astillich-igniti
πŸ’»
avesolovksyy
πŸ› -
avishvat
πŸ› +
avishvat
πŸ›
avivmu
πŸ›
axelbarfod1
πŸ›
b-3-n
πŸ›
balbhadra9
πŸ›
base23de
πŸ›
bergander
πŸ› -
berkam
πŸ’» πŸ› +
berkam
πŸ’» πŸ›
breizh31
πŸ›
caesarkim
πŸ›
carolyujing
πŸ›
cesares-basilico
πŸ›
chrite
πŸ›
cobratbq
πŸ› -
coladict
πŸ› +
coladict
πŸ›
cosmoJFH
πŸ›
cristalp
πŸ›
crunsk
πŸ›
cwholmes
πŸ›
cyberjj999
πŸ›
cyw3
πŸ› -
d1ss0nanz
πŸ› +
d1ss0nanz
πŸ›
dalizi007
πŸ’»
danbrycefairsailcom
πŸ›
dariansanity
πŸ›
darrenmiliband
πŸ›
davidburstrom
πŸ›
dbirkman-paloalto
πŸ› -
deepak-patra
πŸ› +
deepak-patra
πŸ›
dependabot[bot]
πŸ’» πŸ›
dinesh150
πŸ›
diziaq
πŸ›
dreaminpast123
πŸ›
duanyanan
πŸ›
dutt-sanjay
πŸ› -
dylanleung
πŸ› +
dylanleung
πŸ›
dzeigler
πŸ›
ekkirala
πŸ›
emersonmoura
πŸ›
fairy
πŸ›
filiprafalowicz
πŸ’»
foxmason
πŸ› -
frankegabor
πŸ› +
frankegabor
πŸ›
frankl
πŸ›
freafrea
πŸ›
fsapatin
πŸ›
gracia19
πŸ›
guo fei
πŸ›
gurmsc5
πŸ› -
gwilymatgearset
πŸ’» πŸ› +
gwilymatgearset
πŸ’» πŸ›
haigsn
πŸ›
hemanshu070
πŸ›
henrik242
πŸ›
hongpuwu
πŸ›
hvbtup
πŸ’» πŸ›
igniti GmbH
πŸ› -
ilovezfs
πŸ› +
ilovezfs
πŸ›
itaigilo
πŸ›
jakivey32
πŸ›
jbennett2091
πŸ›
jcamerin
πŸ›
jkeener1
πŸ›
jmetertea
πŸ› -
johnra2
πŸ’» +
johnra2
πŸ’»
josemanuelrolon
πŸ’» πŸ›
kabroxiko
πŸ’» πŸ›
karwer
πŸ›
kaulonline
πŸ›
kdaemonv
πŸ›
kenji21
πŸ’» πŸ› -
kfranic
πŸ› +
kfranic
πŸ›
khalidkh
πŸ›
krzyk
πŸ›
lasselindqvist
πŸ›
lgemeinhardt
πŸ›
lihuaib
πŸ›
lonelyma1021
πŸ› -
lpeddy
πŸ› +
lpeddy
πŸ›
lujiefsi
πŸ’»
lukelukes
πŸ’»
lyriccoder
πŸ›
marcelmore
πŸ›
matchbox
πŸ›
matthiaskraaz
πŸ› -
meandonlyme
πŸ› +
meandonlyme
πŸ›
mikesive
πŸ›
milossesic
πŸ›
mriddell95
πŸ›
mrlzh
πŸ›
msloan
πŸ›
mucharlaravalika
πŸ› -
mvenneman
πŸ› +
mvenneman
πŸ›
nareshl119
πŸ›
nicolas-harraudeau-sonarsource
πŸ›
noerremark
πŸ›
novsirion
πŸ›
oggboy
πŸ›
oinume
πŸ› -
orimarko
πŸ’» πŸ› +
orimarko
πŸ’» πŸ›
pallavi agarwal
πŸ›
parksungrin
πŸ›
patpatpat123
πŸ›
patriksevallius
πŸ›
pbrajesh1
πŸ›
phoenix384
πŸ› -
piotrszymanski-sc
πŸ’» +
piotrszymanski-sc
πŸ’»
plan3d
πŸ›
poojasix
πŸ›
prabhushrikant
πŸ›
pujitha8783
πŸ›
r-r-a-j
πŸ›
raghujayjunk
πŸ› -
rajeshveera
πŸ› +
rajeshveera
πŸ›
rajeswarreddy88
πŸ›
recdevs
πŸ›
reudismam
πŸ’» πŸ›
rijkt
πŸ›
rillig-tk
πŸ›
rmohan20
πŸ’» πŸ› -
rxmicro
πŸ› +
rxmicro
πŸ›
ryan-gustafson
πŸ’» πŸ›
sabi0
πŸ›
scais
πŸ›
sebbASF
πŸ›
sergeygorbaty
πŸ’»
shilko2013
πŸ› -
shiomiyan
πŸ“– +
shiomiyan
πŸ“–
simeonKondr
πŸ›
snajberk
πŸ›
sniperrifle2004
πŸ›
snuyanzin
πŸ› πŸ’»
sratz
πŸ›
stonio
πŸ› -
sturton
πŸ’» πŸ› +
sturton
πŸ’» πŸ›
sudharmohan
πŸ›
suruchidawar
πŸ›
svenfinitiv
πŸ›
tashiscool
πŸ›
test-git-hook
πŸ›
testation21
πŸ’» πŸ› -
thanosa
πŸ› +
thanosa
πŸ›
tiandiyixian
πŸ›
tobwoerk
πŸ›
tprouvot
πŸ›
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ› -
tsui
πŸ› +
tsui
πŸ›
winhkey
πŸ›
witherspore
πŸ›
wjljack
πŸ›
wuchiuwong
πŸ›
xingsong
πŸ›
xioayuge
πŸ› -
xnYi9wRezm
πŸ’» πŸ› +
xnYi9wRezm
πŸ’» πŸ›
xuanuy
πŸ›
xyf0921
πŸ›
yalechen-cyw3
πŸ›
yasuharu-sato
πŸ›
zenglian
πŸ›
zgrzyt93
πŸ’» πŸ› -
zh3ng
πŸ› +
zh3ng
πŸ›
zt_soft
πŸ›
ztt79
πŸ›
zzzzfeng
πŸ› From 7c6277c8b8d3f19636be43e280d7ba9548c11c2f Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Mon, 29 Aug 2022 19:05:40 +0200 Subject: [PATCH 122/136] [apex] Update tests (ApexUnitTestClassShouldHaveAsserts, ApexAssertionsShouldIncludeMessage) #4097 --- .../ApexAssertionsShouldIncludeMessage.xml | 68 +++++++++++++++++++ .../ApexUnitTestClassShouldHaveAsserts.xml | 4 +- 2 files changed, 70 insertions(+), 2 deletions(-) diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml index f05e9124da..7a499d556a 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexAssertionsShouldIncludeMessage.xml @@ -116,4 +116,72 @@ public class Foo { } ]]> + + + [apex] Support new Assert class (Apex v56.0) - negative test case #4097 + 9 + 6,12,19,28,34,40,46,52,58 + + diff --git a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml index 28ba29690b..686ffd3cb6 100644 --- a/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml +++ b/pmd-apex/src/test/resources/net/sourceforge/pmd/lang/apex/rule/bestpractices/xml/ApexUnitTestClassShouldHaveAsserts.xml @@ -76,13 +76,13 @@ private class C2_Assignment_Report_Job_Test { #1089 [apex] ApexUnitTestClassShouldHaveAsserts: Verify use of additionalAssertMethodPattern, positive test - (Assert\.\w+|verify\w+) + (MyAssert\.\w+|verify\w+) 0 Date: Mon, 29 Aug 2022 19:06:03 +0200 Subject: [PATCH 123/136] [doc] Update release notes (#4097, #4096) --- docs/pages/release_notes.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 7fcbc0c973..12f1e105b7 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -16,6 +16,8 @@ This is a {{ site.pmd.release_type }} release. ### Fixed Issues +* apex + * [#4096](https://github.com/pmd/pmd/issues/4096): \[apex] ApexAssertionsShouldIncludeMessage and ApexUnitTestClassShouldHaveAsserts: support new Assert class (introduced with Apex v56.0) * java-codestyle * [#4082](https://github.com/pmd/pmd/issues/4082): \[java] UnnecessaryImport false positive for on-demand imports of nested classes @@ -41,6 +43,7 @@ This is a {{ site.pmd.release_type }} release. * [#4081](https://github.com/pmd/pmd/pull/4081): \[apex] Remove Jorje leaks outside `ast` package - [@eklimo](https://github.com/eklimo) * [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) * [#4092](https://github.com/pmd/pmd/pull/4092): \[apex] Implement ApexQualifiableNode for ASTUserEnum - [@aaronhurst-google](https://github.com/aaronhurst-google) +* [#4097](https://github.com/pmd/pmd/pull/4097): \[apex] ApexUnitTestClassShouldHaveAssertsRule: Support new Assert class (Apex v56.0) - [@tprouvot](https://github.com/tprouvot) {% endtocmaker %} From 04b38c3c55a8ddbfd894657194245f791395a23a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Mon, 29 Aug 2022 19:07:55 +0200 Subject: [PATCH 124/136] Update @tprouvot as a contributor --- .all-contributorsrc | 3 ++- docs/pages/pmd/projectdocs/credits.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 87892e6d84..e2edf1e7e1 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -5918,7 +5918,8 @@ "avatar_url": "https://avatars.githubusercontent.com/u/35368290?v=4", "profile": "https://github.com/tprouvot", "contributions": [ - "bug" + "bug", + "code" ] }, { diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index db79a432d7..444b350f40 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -934,7 +934,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
tiandiyixian
πŸ›
tobwoerk
πŸ› -
tprouvot
πŸ› +
tprouvot
πŸ› πŸ’»
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ› From aafd95eb3c2e3e4a45876cefc10030c7fbbbe2f9 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 30 Aug 2022 19:02:42 +0200 Subject: [PATCH 125/136] Add @pacvz as a contributor --- .all-contributorsrc | 9 +++++++++ docs/pages/pmd/projectdocs/credits.md | 21 +++++++++++---------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/.all-contributorsrc b/.all-contributorsrc index 99708783e8..0f9478e0c3 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -6768,6 +6768,15 @@ "code", "financial" ] + }, + { + "login": "pacvz", + "name": "pacvz", + "avatar_url": "https://avatars.githubusercontent.com/u/35453365?v=4", + "profile": "https://github.com/pacvz", + "contributions": [ + "code" + ] } ], "contributorsPerLine": 7, diff --git a/docs/pages/pmd/projectdocs/credits.md b/docs/pages/pmd/projectdocs/credits.md index 37788cf5f7..34eb9d91cd 100644 --- a/docs/pages/pmd/projectdocs/credits.md +++ b/docs/pages/pmd/projectdocs/credits.md @@ -874,91 +874,92 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
oggboy
πŸ›
oinume
πŸ›
orimarko
πŸ’» πŸ› +
pacvz
πŸ’»
pallavi agarwal
πŸ› -
parksungrin
πŸ› +
parksungrin
πŸ›
patpatpat123
πŸ›
patriksevallius
πŸ›
pbrajesh1
πŸ›
phoenix384
πŸ›
piotrszymanski-sc
πŸ’»
plan3d
πŸ› -
poojasix
πŸ› +
poojasix
πŸ›
prabhushrikant
πŸ›
pujitha8783
πŸ›
r-r-a-j
πŸ›
raghujayjunk
πŸ›
rajeshveera
πŸ›
rajeswarreddy88
πŸ› -
recdevs
πŸ› +
recdevs
πŸ›
reudismam
πŸ’» πŸ›
rijkt
πŸ›
rillig-tk
πŸ›
rmohan20
πŸ’» πŸ›
rxmicro
πŸ›
ryan-gustafson
πŸ’» πŸ› -
sabi0
πŸ› +
sabi0
πŸ›
scais
πŸ›
sebbASF
πŸ›
sergeygorbaty
πŸ’»
shilko2013
πŸ›
shiomiyan
πŸ“–
simeonKondr
πŸ› -
snajberk
πŸ› +
snajberk
πŸ›
sniperrifle2004
πŸ›
snuyanzin
πŸ› πŸ’»
sratz
πŸ›
stonio
πŸ›
sturton
πŸ’» πŸ›
sudharmohan
πŸ› -
suruchidawar
πŸ› +
suruchidawar
πŸ›
svenfinitiv
πŸ›
tashiscool
πŸ›
test-git-hook
πŸ›
testation21
πŸ’» πŸ›
thanosa
πŸ›
tiandiyixian
πŸ› -
tobwoerk
πŸ› +
tobwoerk
πŸ›
tprouvot
πŸ›
trentchilders
πŸ›
triandicAnt
πŸ›
trishul14
πŸ›
tsui
πŸ›
winhkey
πŸ› -
witherspore
πŸ› +
witherspore
πŸ›
wjljack
πŸ›
wuchiuwong
πŸ›
xingsong
πŸ›
xioayuge
πŸ›
xnYi9wRezm
πŸ’» πŸ›
xuanuy
πŸ› -
xyf0921
πŸ› +
xyf0921
πŸ›
yalechen-cyw3
πŸ›
yasuharu-sato
πŸ›
zenglian
πŸ›
zgrzyt93
πŸ’» πŸ›
zh3ng
πŸ›
zt_soft
πŸ› -
ztt79
πŸ› +
ztt79
πŸ›
zzzzfeng
πŸ›
ÁrpÑd MagosÑnyi
πŸ›
任贡杰
πŸ› From 504d2ad5f9c4d9a89d9ef4c03e18709bec1b46af Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Tue, 30 Aug 2022 19:04:08 +0200 Subject: [PATCH 126/136] [doc] Update release notes (#4095) --- docs/pages/release_notes.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index b8f8783555..b90bc890c5 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -20,5 +20,7 @@ This is a {{ site.pmd.release_type }} release. ### External Contributions +* [#4095](https://github.com/pmd/pmd/pull/4095): \[core] CPD: Added begin and end token to XML reports - [@pacvz](https://github.com/pacvz) + {% endtocmaker %} From fe65a6ba9874c420de04b7cb8f686eeca812ad08 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 18:34:48 +0200 Subject: [PATCH 127/136] Bump pmd-designer from 6.37.0 to 6.49.0 --- docs/pages/release_notes.md | 5 +++++ pom.xml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index acd1d64786..88a247d79b 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,6 +14,11 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy +#### Updated PMD Designer + +This PMD release ships a new version of the pmd-designer. +For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designer/releases/tag/6.49.0). + ### Fixed Issues * apex diff --git a/pom.xml b/pom.xml index 5b215e903e..765ba24625 100644 --- a/pom.xml +++ b/pom.xml @@ -108,7 +108,7 @@ 18 - 6.37.0 + 6.49.0 From f225ee5ff161d31d2986a6c88d621d1a44012cfd Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 18:40:07 +0200 Subject: [PATCH 128/136] Fix checkstyle, PMD --- .../java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java | 4 ---- .../src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java index 48fcb436af..a457e3ef04 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/lang/apex/ast/ASTUserEnum.java @@ -18,10 +18,6 @@ public final class ASTUserEnum extends BaseApexClass { return visitor.visit(this, data); } - public ASTModifierNode getModifiers() { - return getFirstChildOfType(ASTModifierNode.class); - } - @Override public ApexQualifiedName getQualifiedName() { if (qname == null) { diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java index eb86f11d9e..9ca21e38a8 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/PmdAnalysisTest.java @@ -33,7 +33,7 @@ import net.sourceforge.pmd.reporting.ReportStats; /** * @author ClΓ©ment Fournier */ -class PmdAnalysisTest { +public class PmdAnalysisTest { @Test void testPmdAnalysisWithEmptyConfig() { From eba30d1897b47a12719f131b93ac6495921cca6a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 18:40:44 +0200 Subject: [PATCH 129/136] Cleanup --- .../internal/LanguageAwareDataSource.java | 42 ------------------- 1 file changed, 42 deletions(-) delete mode 100644 pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java b/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java deleted file mode 100644 index 407001e102..0000000000 --- a/pmd-core/src/main/java/net/sourceforge/pmd/util/datasource/internal/LanguageAwareDataSource.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * BSD-style license; for more info see http://pmd.sourceforge.net/license.html - */ - -package net.sourceforge.pmd.util.datasource.internal; - -import java.io.IOException; -import java.io.InputStream; - -import net.sourceforge.pmd.annotation.InternalApi; -import net.sourceforge.pmd.lang.LanguageVersion; -import net.sourceforge.pmd.util.datasource.DataSource; - -@InternalApi -public class LanguageAwareDataSource implements DataSource { - private final DataSource base; // delegate DataSource methods to this - private final LanguageVersion version; - - public LanguageAwareDataSource(DataSource base, LanguageVersion version) { - this.base = base; - this.version = version; - } - - public LanguageVersion getLanguageVersion() { - return version; - } - - @Override - public InputStream getInputStream() throws IOException { - return base.getInputStream(); - } - - @Override - public String getNiceFileName(boolean shortNames, String inputFileName) { - return base.getNiceFileName(shortNames, inputFileName); - } - - @Override - public void close() throws IOException { - base.close(); - } -} From 232949566a3ad556aefbc24bef495312c78c704a Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 19:07:21 +0200 Subject: [PATCH 130/136] Prepare pmd release 6.49.0 --- docs/_config.yml | 2 +- docs/pages/next_major_development.md | 17 +++++++++++++++++ docs/pages/release_notes.md | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/_config.yml b/docs/_config.yml index 75b1ae16eb..3286fd80dd 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,7 +1,7 @@ repository: pmd/pmd pmd: - version: 6.49.0-SNAPSHOT + version: 6.49.0 previous_version: 6.48.0 date: 31-August-2022 release_type: minor diff --git a/docs/pages/next_major_development.md b/docs/pages/next_major_development.md index 5cc83709eb..45333ab4e7 100644 --- a/docs/pages/next_major_development.md +++ b/docs/pages/next_major_development.md @@ -125,6 +125,23 @@ the breaking API changes will be performed in 7.0.0. an API is tagged as `@Deprecated` or not in the latest minor release. During the development of 7.0.0, we may decide to remove some APIs that were not tagged as deprecated, though we'll try to avoid it." %} +#### 6.49.0 + +##### Deprecated API + +* In order to reduce the dependency on Apex Jorje classes, the following methods have been deprecated. + These methods all leaked internal Jorje enums. These enums have been replaced now by enums the + PMD's AST package. + * {% jdoc !!apex::lang.apex.ast.ASTAssignmentExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTBinaryExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTBooleanExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTPostfixExpression#getOperator() %} + * {% jdoc !!apex::lang.apex.ast.ASTPrefixExpression#getOperator() %} + + All these classes have now a new `getOp()` method. Existing code should be refactored to use this method instead. + It returns the new enums, like {% jdoc apex::lang.apex.ast.AssignmentOperator %}, and avoids + the dependency to Jorje. + #### 6.48.0 ##### CPD CLI diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 88a247d79b..3c5a1007bd 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -54,5 +54,10 @@ For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designe * [#4097](https://github.com/pmd/pmd/pull/4097): \[apex] ApexUnitTestClassShouldHaveAssertsRule: Support new Assert class (Apex v56.0) - [@tprouvot](https://github.com/tprouvot) * [#4104](https://github.com/pmd/pmd/pull/4104): \[doc] Add MegaLinter in the list of integrations - [@nvuillam](https://github.com/nvuillam) +### Stats +* 49 commits +* 10 closed tickets & PRs +* Days since last release: 32 + {% endtocmaker %} From 4b4ebabc2d5ed534c2657882b23d4ba2dd9da8d3 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 19:19:22 +0200 Subject: [PATCH 131/136] [maven-release-plugin] prepare release pmd_releases/6.49.0 --- pmd-apex-jorje/pom.xml | 2 +- pmd-apex/pom.xml | 2 +- pmd-core/pom.xml | 2 +- pmd-cpp/pom.xml | 2 +- pmd-cs/pom.xml | 2 +- pmd-dart/pom.xml | 2 +- pmd-dist/pom.xml | 2 +- pmd-doc/pom.xml | 2 +- pmd-fortran/pom.xml | 2 +- pmd-gherkin/pom.xml | 2 +- pmd-go/pom.xml | 2 +- pmd-groovy/pom.xml | 2 +- pmd-html/pom.xml | 2 +- pmd-java/pom.xml | 2 +- pmd-java8/pom.xml | 2 +- pmd-javascript/pom.xml | 2 +- pmd-jsp/pom.xml | 2 +- pmd-kotlin/pom.xml | 2 +- pmd-lang-test/pom.xml | 2 +- pmd-lua/pom.xml | 2 +- pmd-matlab/pom.xml | 2 +- pmd-modelica/pom.xml | 2 +- pmd-objectivec/pom.xml | 2 +- pmd-perl/pom.xml | 2 +- pmd-php/pom.xml | 2 +- pmd-plsql/pom.xml | 2 +- pmd-python/pom.xml | 2 +- pmd-ruby/pom.xml | 2 +- pmd-scala-modules/pmd-scala-common/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.12/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.13/pom.xml | 2 +- pmd-scala/pom.xml | 2 +- pmd-swift/pom.xml | 2 +- pmd-test-schema/pom.xml | 2 +- pmd-test/pom.xml | 2 +- pmd-visualforce/pom.xml | 2 +- pmd-vm/pom.xml | 2 +- pmd-xml/pom.xml | 2 +- pom.xml | 6 +++--- 39 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index 9d82fcc6e2..3ba917c087 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-apex/pom.xml b/pmd-apex/pom.xml index 64d14d5a86..02e6a9ad75 100644 --- a/pmd-apex/pom.xml +++ b/pmd-apex/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index f43367694a..56ef59bdc1 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-cpp/pom.xml b/pmd-cpp/pom.xml index 6f3d3afcee..f931d5a2bd 100644 --- a/pmd-cpp/pom.xml +++ b/pmd-cpp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-cs/pom.xml b/pmd-cs/pom.xml index 345f57f90b..1397723772 100644 --- a/pmd-cs/pom.xml +++ b/pmd-cs/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-dart/pom.xml b/pmd-dart/pom.xml index 8989f246ce..5414173e27 100644 --- a/pmd-dart/pom.xml +++ b/pmd-dart/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-dist/pom.xml b/pmd-dist/pom.xml index c5912ee704..ef0437a867 100644 --- a/pmd-dist/pom.xml +++ b/pmd-dist/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-doc/pom.xml b/pmd-doc/pom.xml index d53987ee65..6a5483fee1 100644 --- a/pmd-doc/pom.xml +++ b/pmd-doc/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-fortran/pom.xml b/pmd-fortran/pom.xml index c092b17d49..c832b38f7e 100644 --- a/pmd-fortran/pom.xml +++ b/pmd-fortran/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-gherkin/pom.xml b/pmd-gherkin/pom.xml index 047f06705d..a5b3020ce3 100644 --- a/pmd-gherkin/pom.xml +++ b/pmd-gherkin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-go/pom.xml b/pmd-go/pom.xml index 8fff13187f..120fe6529e 100644 --- a/pmd-go/pom.xml +++ b/pmd-go/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-groovy/pom.xml b/pmd-groovy/pom.xml index 4f04a88d66..6211be3231 100644 --- a/pmd-groovy/pom.xml +++ b/pmd-groovy/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-html/pom.xml b/pmd-html/pom.xml index 70c45debaa..18dc127f71 100644 --- a/pmd-html/pom.xml +++ b/pmd-html/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-java/pom.xml b/pmd-java/pom.xml index 1f0ca4dae9..aa9923878f 100644 --- a/pmd-java/pom.xml +++ b/pmd-java/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-java8/pom.xml b/pmd-java8/pom.xml index 6a4c97c7c6..0184e54369 100644 --- a/pmd-java8/pom.xml +++ b/pmd-java8/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-javascript/pom.xml b/pmd-javascript/pom.xml index 0c58e2aef2..fbfefc29af 100644 --- a/pmd-javascript/pom.xml +++ b/pmd-javascript/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-jsp/pom.xml b/pmd-jsp/pom.xml index 14f1f6d76b..5dc7b6e7a3 100644 --- a/pmd-jsp/pom.xml +++ b/pmd-jsp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-kotlin/pom.xml b/pmd-kotlin/pom.xml index 125ac5cdc0..8c035d3b64 100644 --- a/pmd-kotlin/pom.xml +++ b/pmd-kotlin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-lang-test/pom.xml b/pmd-lang-test/pom.xml index f53f195b22..5c2f2bd1a1 100644 --- a/pmd-lang-test/pom.xml +++ b/pmd-lang-test/pom.xml @@ -12,7 +12,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-lua/pom.xml b/pmd-lua/pom.xml index f342207fd0..591a892838 100644 --- a/pmd-lua/pom.xml +++ b/pmd-lua/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-matlab/pom.xml b/pmd-matlab/pom.xml index be40507861..d7bceb2345 100644 --- a/pmd-matlab/pom.xml +++ b/pmd-matlab/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-modelica/pom.xml b/pmd-modelica/pom.xml index 72b7a50b8f..8c91ac990c 100644 --- a/pmd-modelica/pom.xml +++ b/pmd-modelica/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-objectivec/pom.xml b/pmd-objectivec/pom.xml index 4aee2cf7dc..ecd8ca82e5 100644 --- a/pmd-objectivec/pom.xml +++ b/pmd-objectivec/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-perl/pom.xml b/pmd-perl/pom.xml index 9a44d3d7d7..c099d62ab8 100644 --- a/pmd-perl/pom.xml +++ b/pmd-perl/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-php/pom.xml b/pmd-php/pom.xml index af01ee2f6f..603ad424ea 100644 --- a/pmd-php/pom.xml +++ b/pmd-php/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-plsql/pom.xml b/pmd-plsql/pom.xml index b3cf518cce..8f2c4ac37d 100644 --- a/pmd-plsql/pom.xml +++ b/pmd-plsql/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-python/pom.xml b/pmd-python/pom.xml index 9c30438d09..ae4193f020 100644 --- a/pmd-python/pom.xml +++ b/pmd-python/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-ruby/pom.xml b/pmd-ruby/pom.xml index 62b5d6b15b..5d02310c54 100644 --- a/pmd-ruby/pom.xml +++ b/pmd-ruby/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-scala-modules/pmd-scala-common/pom.xml b/pmd-scala-modules/pmd-scala-common/pom.xml index 14ac1f8367..0540eadb1c 100644 --- a/pmd-scala-modules/pmd-scala-common/pom.xml +++ b/pmd-scala-modules/pmd-scala-common/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../../pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.12/pom.xml b/pmd-scala-modules/pmd-scala_2.12/pom.xml index 61ac0b7d84..d10c138b51 100644 --- a/pmd-scala-modules/pmd-scala_2.12/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.12/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.49.0-SNAPSHOT + 6.49.0 ../pmd-scala-common/pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.13/pom.xml b/pmd-scala-modules/pmd-scala_2.13/pom.xml index 6bf91dcd09..3196d78233 100644 --- a/pmd-scala-modules/pmd-scala_2.13/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.13/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.49.0-SNAPSHOT + 6.49.0 ../pmd-scala-common/pom.xml diff --git a/pmd-scala/pom.xml b/pmd-scala/pom.xml index c12939498b..1ccb15045b 100644 --- a/pmd-scala/pom.xml +++ b/pmd-scala/pom.xml @@ -9,7 +9,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-swift/pom.xml b/pmd-swift/pom.xml index 43699a38b0..b0955b67be 100644 --- a/pmd-swift/pom.xml +++ b/pmd-swift/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-test-schema/pom.xml b/pmd-test-schema/pom.xml index 5bb6d34c03..fcd307894c 100644 --- a/pmd-test-schema/pom.xml +++ b/pmd-test-schema/pom.xml @@ -11,7 +11,7 @@ pmd net.sourceforge.pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-test/pom.xml b/pmd-test/pom.xml index 50e48f3a30..b1fd383293 100644 --- a/pmd-test/pom.xml +++ b/pmd-test/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-visualforce/pom.xml b/pmd-visualforce/pom.xml index f66ecc00cb..7c4f10cd1d 100644 --- a/pmd-visualforce/pom.xml +++ b/pmd-visualforce/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-vm/pom.xml b/pmd-vm/pom.xml index 597785ae98..092b65005f 100644 --- a/pmd-vm/pom.xml +++ b/pmd-vm/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pmd-xml/pom.xml b/pmd-xml/pom.xml index 3b5e97f73e..4d88ce6208 100644 --- a/pmd-xml/pom.xml +++ b/pmd-xml/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 ../pom.xml diff --git a/pom.xml b/pom.xml index 765ba24625..18e7061b14 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 net.sourceforge.pmd pmd - 6.49.0-SNAPSHOT + 6.49.0 pom PMD @@ -55,7 +55,7 @@ scm:git:git://github.com/pmd/pmd.git scm:git:ssh://git@github.com/pmd/pmd.git https://github.com/pmd/pmd - HEAD + pmd_releases/6.49.0 @@ -76,7 +76,7 @@ - 2022-07-30T09:35:57Z + 2022-08-31T17:07:30Z 7 From 90663d746b925d0aad9d078ddbdf7fd11b2ffbfe Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 19:19:27 +0200 Subject: [PATCH 132/136] [maven-release-plugin] prepare for next development iteration --- pmd-apex-jorje/pom.xml | 2 +- pmd-apex/pom.xml | 2 +- pmd-core/pom.xml | 2 +- pmd-cpp/pom.xml | 2 +- pmd-cs/pom.xml | 2 +- pmd-dart/pom.xml | 2 +- pmd-dist/pom.xml | 2 +- pmd-doc/pom.xml | 2 +- pmd-fortran/pom.xml | 2 +- pmd-gherkin/pom.xml | 2 +- pmd-go/pom.xml | 2 +- pmd-groovy/pom.xml | 2 +- pmd-html/pom.xml | 2 +- pmd-java/pom.xml | 2 +- pmd-java8/pom.xml | 2 +- pmd-javascript/pom.xml | 2 +- pmd-jsp/pom.xml | 2 +- pmd-kotlin/pom.xml | 2 +- pmd-lang-test/pom.xml | 2 +- pmd-lua/pom.xml | 2 +- pmd-matlab/pom.xml | 2 +- pmd-modelica/pom.xml | 2 +- pmd-objectivec/pom.xml | 2 +- pmd-perl/pom.xml | 2 +- pmd-php/pom.xml | 2 +- pmd-plsql/pom.xml | 2 +- pmd-python/pom.xml | 2 +- pmd-ruby/pom.xml | 2 +- pmd-scala-modules/pmd-scala-common/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.12/pom.xml | 2 +- pmd-scala-modules/pmd-scala_2.13/pom.xml | 2 +- pmd-scala/pom.xml | 2 +- pmd-swift/pom.xml | 2 +- pmd-test-schema/pom.xml | 2 +- pmd-test/pom.xml | 2 +- pmd-visualforce/pom.xml | 2 +- pmd-vm/pom.xml | 2 +- pmd-xml/pom.xml | 2 +- pom.xml | 6 +++--- 39 files changed, 41 insertions(+), 41 deletions(-) diff --git a/pmd-apex-jorje/pom.xml b/pmd-apex-jorje/pom.xml index 3ba917c087..e39ef4b13d 100644 --- a/pmd-apex-jorje/pom.xml +++ b/pmd-apex-jorje/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-apex/pom.xml b/pmd-apex/pom.xml index 02e6a9ad75..0302dd9be0 100644 --- a/pmd-apex/pom.xml +++ b/pmd-apex/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-core/pom.xml b/pmd-core/pom.xml index 56ef59bdc1..2cfe77ea15 100644 --- a/pmd-core/pom.xml +++ b/pmd-core/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-cpp/pom.xml b/pmd-cpp/pom.xml index f931d5a2bd..dc6312c4d8 100644 --- a/pmd-cpp/pom.xml +++ b/pmd-cpp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-cs/pom.xml b/pmd-cs/pom.xml index 1397723772..39cb5a5124 100644 --- a/pmd-cs/pom.xml +++ b/pmd-cs/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-dart/pom.xml b/pmd-dart/pom.xml index 5414173e27..62ecfa06d7 100644 --- a/pmd-dart/pom.xml +++ b/pmd-dart/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-dist/pom.xml b/pmd-dist/pom.xml index ef0437a867..5bf7cf2aa5 100644 --- a/pmd-dist/pom.xml +++ b/pmd-dist/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-doc/pom.xml b/pmd-doc/pom.xml index 6a5483fee1..c215c8b406 100644 --- a/pmd-doc/pom.xml +++ b/pmd-doc/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-fortran/pom.xml b/pmd-fortran/pom.xml index c832b38f7e..d6017d7530 100644 --- a/pmd-fortran/pom.xml +++ b/pmd-fortran/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-gherkin/pom.xml b/pmd-gherkin/pom.xml index a5b3020ce3..c134fc08fa 100644 --- a/pmd-gherkin/pom.xml +++ b/pmd-gherkin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-go/pom.xml b/pmd-go/pom.xml index 120fe6529e..387bbcf234 100644 --- a/pmd-go/pom.xml +++ b/pmd-go/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-groovy/pom.xml b/pmd-groovy/pom.xml index 6211be3231..3c72a3acfc 100644 --- a/pmd-groovy/pom.xml +++ b/pmd-groovy/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-html/pom.xml b/pmd-html/pom.xml index 18dc127f71..bbd5c0a746 100644 --- a/pmd-html/pom.xml +++ b/pmd-html/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-java/pom.xml b/pmd-java/pom.xml index aa9923878f..1999933e18 100644 --- a/pmd-java/pom.xml +++ b/pmd-java/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-java8/pom.xml b/pmd-java8/pom.xml index 0184e54369..dfbfe23d76 100644 --- a/pmd-java8/pom.xml +++ b/pmd-java8/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-javascript/pom.xml b/pmd-javascript/pom.xml index fbfefc29af..a88e165bd3 100644 --- a/pmd-javascript/pom.xml +++ b/pmd-javascript/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-jsp/pom.xml b/pmd-jsp/pom.xml index 5dc7b6e7a3..3ec09898c0 100644 --- a/pmd-jsp/pom.xml +++ b/pmd-jsp/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-kotlin/pom.xml b/pmd-kotlin/pom.xml index 8c035d3b64..26d6620be5 100644 --- a/pmd-kotlin/pom.xml +++ b/pmd-kotlin/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-lang-test/pom.xml b/pmd-lang-test/pom.xml index 5c2f2bd1a1..3bb187261e 100644 --- a/pmd-lang-test/pom.xml +++ b/pmd-lang-test/pom.xml @@ -12,7 +12,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-lua/pom.xml b/pmd-lua/pom.xml index 591a892838..b93a17f046 100644 --- a/pmd-lua/pom.xml +++ b/pmd-lua/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-matlab/pom.xml b/pmd-matlab/pom.xml index d7bceb2345..ba5076d6e9 100644 --- a/pmd-matlab/pom.xml +++ b/pmd-matlab/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-modelica/pom.xml b/pmd-modelica/pom.xml index 8c91ac990c..4d09398f22 100644 --- a/pmd-modelica/pom.xml +++ b/pmd-modelica/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-objectivec/pom.xml b/pmd-objectivec/pom.xml index ecd8ca82e5..6dd1bbe61f 100644 --- a/pmd-objectivec/pom.xml +++ b/pmd-objectivec/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-perl/pom.xml b/pmd-perl/pom.xml index c099d62ab8..5342902fd5 100644 --- a/pmd-perl/pom.xml +++ b/pmd-perl/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-php/pom.xml b/pmd-php/pom.xml index 603ad424ea..983c228b54 100644 --- a/pmd-php/pom.xml +++ b/pmd-php/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-plsql/pom.xml b/pmd-plsql/pom.xml index 8f2c4ac37d..31e821cd95 100644 --- a/pmd-plsql/pom.xml +++ b/pmd-plsql/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-python/pom.xml b/pmd-python/pom.xml index ae4193f020..8e7a6451d9 100644 --- a/pmd-python/pom.xml +++ b/pmd-python/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-ruby/pom.xml b/pmd-ruby/pom.xml index 5d02310c54..f1c8297b34 100644 --- a/pmd-ruby/pom.xml +++ b/pmd-ruby/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-scala-modules/pmd-scala-common/pom.xml b/pmd-scala-modules/pmd-scala-common/pom.xml index 0540eadb1c..0997d90d5d 100644 --- a/pmd-scala-modules/pmd-scala-common/pom.xml +++ b/pmd-scala-modules/pmd-scala-common/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../../pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.12/pom.xml b/pmd-scala-modules/pmd-scala_2.12/pom.xml index d10c138b51..9d8b80cbbd 100644 --- a/pmd-scala-modules/pmd-scala_2.12/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.12/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.49.0 + 6.50.0-SNAPSHOT ../pmd-scala-common/pom.xml diff --git a/pmd-scala-modules/pmd-scala_2.13/pom.xml b/pmd-scala-modules/pmd-scala_2.13/pom.xml index 3196d78233..ae57caed29 100644 --- a/pmd-scala-modules/pmd-scala_2.13/pom.xml +++ b/pmd-scala-modules/pmd-scala_2.13/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd-scala-common - 6.49.0 + 6.50.0-SNAPSHOT ../pmd-scala-common/pom.xml diff --git a/pmd-scala/pom.xml b/pmd-scala/pom.xml index 1ccb15045b..5b199fa762 100644 --- a/pmd-scala/pom.xml +++ b/pmd-scala/pom.xml @@ -9,7 +9,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-swift/pom.xml b/pmd-swift/pom.xml index b0955b67be..a363e59183 100644 --- a/pmd-swift/pom.xml +++ b/pmd-swift/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-test-schema/pom.xml b/pmd-test-schema/pom.xml index fcd307894c..37e4c2de71 100644 --- a/pmd-test-schema/pom.xml +++ b/pmd-test-schema/pom.xml @@ -11,7 +11,7 @@ pmd net.sourceforge.pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-test/pom.xml b/pmd-test/pom.xml index b1fd383293..6ddeaaff3a 100644 --- a/pmd-test/pom.xml +++ b/pmd-test/pom.xml @@ -8,7 +8,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-visualforce/pom.xml b/pmd-visualforce/pom.xml index 7c4f10cd1d..61c823bda1 100644 --- a/pmd-visualforce/pom.xml +++ b/pmd-visualforce/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-vm/pom.xml b/pmd-vm/pom.xml index 092b65005f..399246c0d5 100644 --- a/pmd-vm/pom.xml +++ b/pmd-vm/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pmd-xml/pom.xml b/pmd-xml/pom.xml index 4d88ce6208..ecb77817dc 100644 --- a/pmd-xml/pom.xml +++ b/pmd-xml/pom.xml @@ -7,7 +7,7 @@ net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT ../pom.xml diff --git a/pom.xml b/pom.xml index 18e7061b14..bfe3fe4c34 100644 --- a/pom.xml +++ b/pom.xml @@ -3,7 +3,7 @@ 4.0.0 net.sourceforge.pmd pmd - 6.49.0 + 6.50.0-SNAPSHOT pom PMD @@ -55,7 +55,7 @@ scm:git:git://github.com/pmd/pmd.git scm:git:ssh://git@github.com/pmd/pmd.git https://github.com/pmd/pmd - pmd_releases/6.49.0 + HEAD @@ -76,7 +76,7 @@ - 2022-08-31T17:07:30Z + 2022-08-31T17:19:27Z 7 From 76be7ba6690e20a4caee3c0d6dd13c6322243586 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Wed, 31 Aug 2022 19:20:51 +0200 Subject: [PATCH 133/136] Prepare next development version [skip ci] --- docs/_config.yml | 6 ++-- docs/pages/release_notes.md | 39 -------------------- docs/pages/release_notes_old.md | 63 +++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 42 deletions(-) diff --git a/docs/_config.yml b/docs/_config.yml index 3286fd80dd..a2bce8a5ea 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,9 +1,9 @@ repository: pmd/pmd pmd: - version: 6.49.0 - previous_version: 6.48.0 - date: 31-August-2022 + version: 6.50.0-SNAPSHOT + previous_version: 6.49.0 + date: 30-September-2022 release_type: minor # release types: major, minor, bugfix diff --git a/docs/pages/release_notes.md b/docs/pages/release_notes.md index 3c5a1007bd..b8f8783555 100644 --- a/docs/pages/release_notes.md +++ b/docs/pages/release_notes.md @@ -14,50 +14,11 @@ This is a {{ site.pmd.release_type }} release. ### New and noteworthy -#### Updated PMD Designer - -This PMD release ships a new version of the pmd-designer. -For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designer/releases/tag/6.49.0). - ### Fixed Issues -* apex - * [#4096](https://github.com/pmd/pmd/issues/4096): \[apex] ApexAssertionsShouldIncludeMessage and ApexUnitTestClassShouldHaveAsserts: support new Assert class (introduced with Apex v56.0) -* core - * [#3970](https://github.com/pmd/pmd/issues/3970): \[core] FileCollector.addFile ignores language parameter -* java-codestyle - * [#4082](https://github.com/pmd/pmd/issues/4082): \[java] UnnecessaryImport false positive for on-demand imports of nested classes - ### API Changes -#### Deprecated API - -* In order to reduce the dependency on Apex Jorje classes, the following methods have been deprecated. - These methods all leaked internal Jorje enums. These enums have been replaced now by enums the - PMD's AST package. - * {% jdoc !!apex::lang.apex.ast.ASTAssignmentExpression#getOperator() %} - * {% jdoc !!apex::lang.apex.ast.ASTBinaryExpression#getOperator() %} - * {% jdoc !!apex::lang.apex.ast.ASTBooleanExpression#getOperator() %} - * {% jdoc !!apex::lang.apex.ast.ASTPostfixExpression#getOperator() %} - * {% jdoc !!apex::lang.apex.ast.ASTPrefixExpression#getOperator() %} - - All these classes have now a new `getOp()` method. Existing code should be refactored to use this method instead. - It returns the new enums, like {% jdoc apex::lang.apex.ast.AssignmentOperator %}, and avoids - the dependency to Jorje. - ### External Contributions -* [#4081](https://github.com/pmd/pmd/pull/4081): \[apex] Remove Jorje leaks outside `ast` package - [@eklimo](https://github.com/eklimo) -* [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) -* [#4092](https://github.com/pmd/pmd/pull/4092): \[apex] Implement ApexQualifiableNode for ASTUserEnum - [@aaronhurst-google](https://github.com/aaronhurst-google) -* [#4095](https://github.com/pmd/pmd/pull/4095): \[core] CPD: Added begin and end token to XML reports - [@pacvz](https://github.com/pacvz) -* [#4097](https://github.com/pmd/pmd/pull/4097): \[apex] ApexUnitTestClassShouldHaveAssertsRule: Support new Assert class (Apex v56.0) - [@tprouvot](https://github.com/tprouvot) -* [#4104](https://github.com/pmd/pmd/pull/4104): \[doc] Add MegaLinter in the list of integrations - [@nvuillam](https://github.com/nvuillam) - -### Stats -* 49 commits -* 10 closed tickets & PRs -* Days since last release: 32 - {% endtocmaker %} diff --git a/docs/pages/release_notes_old.md b/docs/pages/release_notes_old.md index cd56c15d74..932e00b8b9 100644 --- a/docs/pages/release_notes_old.md +++ b/docs/pages/release_notes_old.md @@ -5,6 +5,69 @@ permalink: pmd_release_notes_old.html Previous versions of PMD can be downloaded here: https://github.com/pmd/pmd/releases +## 31-August-2022 - 6.49.0 + +The PMD team is pleased to announce PMD 6.49.0. + +This is a minor release. + +### Table Of Contents + +* [New and noteworthy](#new-and-noteworthy) + * [Updated PMD Designer](#updated-pmd-designer) +* [Fixed Issues](#fixed-issues) +* [API Changes](#api-changes) + * [Deprecated API](#deprecated-api) +* [External Contributions](#external-contributions) +* [Stats](#stats) + +### New and noteworthy + +#### Updated PMD Designer + +This PMD release ships a new version of the pmd-designer. +For the changes, see [PMD Designer Changelog](https://github.com/pmd/pmd-designer/releases/tag/6.49.0). + +### Fixed Issues + +* apex + * [#4096](https://github.com/pmd/pmd/issues/4096): \[apex] ApexAssertionsShouldIncludeMessage and ApexUnitTestClassShouldHaveAsserts: support new Assert class (introduced with Apex v56.0) +* core + * [#3970](https://github.com/pmd/pmd/issues/3970): \[core] FileCollector.addFile ignores language parameter +* java-codestyle + * [#4082](https://github.com/pmd/pmd/issues/4082): \[java] UnnecessaryImport false positive for on-demand imports of nested classes + +### API Changes + +#### Deprecated API + +* In order to reduce the dependency on Apex Jorje classes, the following methods have been deprecated. + These methods all leaked internal Jorje enums. These enums have been replaced now by enums the + PMD's AST package. + * ASTAssignmentExpression#getOperator + * ASTBinaryExpression#getOperator + * ASTBooleanExpression#getOperator + * ASTPostfixExpression#getOperator + * ASTPrefixExpression#getOperator + + All these classes have now a new `getOp()` method. Existing code should be refactored to use this method instead. + It returns the new enums, like AssignmentOperator, and avoids + the dependency to Jorje. + +### External Contributions + +* [#4081](https://github.com/pmd/pmd/pull/4081): \[apex] Remove Jorje leaks outside `ast` package - [@eklimo](https://github.com/eklimo) +* [#4083](https://github.com/pmd/pmd/pull/4083): \[java] UnnecessaryImport false positive for on-demand imports of nested classes (fix for #4082) - [@abyss638](https://github.com/abyss638) +* [#4092](https://github.com/pmd/pmd/pull/4092): \[apex] Implement ApexQualifiableNode for ASTUserEnum - [@aaronhurst-google](https://github.com/aaronhurst-google) +* [#4095](https://github.com/pmd/pmd/pull/4095): \[core] CPD: Added begin and end token to XML reports - [@pacvz](https://github.com/pacvz) +* [#4097](https://github.com/pmd/pmd/pull/4097): \[apex] ApexUnitTestClassShouldHaveAssertsRule: Support new Assert class (Apex v56.0) - [@tprouvot](https://github.com/tprouvot) +* [#4104](https://github.com/pmd/pmd/pull/4104): \[doc] Add MegaLinter in the list of integrations - [@nvuillam](https://github.com/nvuillam) + +### Stats +* 49 commits +* 10 closed tickets & PRs +* Days since last release: 32 + ## 30-July-2022 - 6.48.0 The PMD team is pleased to announce PMD 6.48.0. From f3dbdb700cc0ef592b31cab299a38c08ba3f6d4d Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 9 Sep 2022 16:18:29 +0200 Subject: [PATCH 134/136] Fixups from #4044 --- .../net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java | 4 +--- .../net/sourceforge/pmd/lang/document/RootTextDocument.java | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java index b1a5c8f0d1..d598bff263 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStream.java @@ -18,8 +18,6 @@ import net.sourceforge.pmd.lang.document.TextRegion; */ public final class CharStream { - private static final EOFException EOF = new EOFException(); - private final JavaccTokenDocument tokenDoc; private final TextDocument textDoc; private final Chars chars; @@ -54,7 +52,7 @@ public final class CharStream { */ public char readChar() throws EOFException { if (curOffset == chars.length()) { - throw EOF; + throw new EOFException(); } return chars.charAt(curOffset++); } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java index 8dc79889ec..be09e85419 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/document/RootTextDocument.java @@ -126,7 +126,7 @@ final class RootTextDocument extends BaseCloseable implements TextDocument { private static final String NOT_IN_RANGE = "Region [start=%d, end=%d[ is not in range of this document (length %d)"; private static final String INVALID_LINE_RANGE = "Line range %d..%d is not in range of this document (%d lines) (line numbers are 1-based)"; - private static final String INVALID_OFFSET = "Offset %d is not in range of this document (length %d) (line numbers are 0-based)"; + private static final String INVALID_OFFSET = "Offset %d is not in range of this document (length %d) (offsets are 0-based)"; static IndexOutOfBoundsException invalidLineRange(int start, int end, int numLines) { return new IndexOutOfBoundsException(String.format(INVALID_LINE_RANGE, start, end, numLines)); From 2072c8e08b7d2997cfcab778cbf423f7ccf586b5 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 10 Sep 2022 20:07:33 +0200 Subject: [PATCH 135/136] Fix javadoc --- .../net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java b/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java index bc9f739ff8..927e6572d3 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/lang/LanguageVersionDiscoverer.java @@ -43,10 +43,6 @@ public class LanguageVersionDiscoverer { /** * Build a new instance with no forced version. - * - * @param forcedVersion If non-null, all files should be assigned this version. - * The methods of this class still work as usual and do not - * care about the forced language version. */ public LanguageVersionDiscoverer(LanguageRegistry registry) { this(registry, null); From 8cc3e47ce410e7608c5b2639dd201860b40bc349 Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Sat, 10 Sep 2022 20:17:40 +0200 Subject: [PATCH 136/136] Fix compile errors --- .../sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java | 4 ++-- .../pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java index 40e1740657..6e4662e5cd 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/CharStreamTest.java @@ -13,14 +13,14 @@ import java.util.Collections; import org.junit.Test; -import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.DummyLanguageModule; import net.sourceforge.pmd.lang.LanguageVersion; import net.sourceforge.pmd.lang.ast.impl.javacc.JavaccTokenDocument.TokenDocumentBehavior; import net.sourceforge.pmd.lang.document.TextDocument; public class CharStreamTest { - private LanguageVersion dummyVersion = LanguageRegistry.getDefaultLanguage().getDefaultVersion(); + private LanguageVersion dummyVersion = DummyLanguageModule.getInstance().getDefaultVersion(); @Test public void testReadZeroChars() throws IOException { diff --git a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java index 1d43033bd2..6c0f5ec6e9 100644 --- a/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java +++ b/pmd-core/src/test/java/net/sourceforge/pmd/lang/ast/impl/javacc/JavaEscapeReaderTest.java @@ -10,7 +10,7 @@ import java.io.IOException; import org.junit.Test; -import net.sourceforge.pmd.lang.LanguageRegistry; +import net.sourceforge.pmd.lang.DummyLanguageModule; import net.sourceforge.pmd.lang.document.Chars; import net.sourceforge.pmd.lang.document.TextDocument; @@ -18,7 +18,7 @@ import net.sourceforge.pmd.lang.document.TextDocument; public class JavaEscapeReaderTest { public TextDocument readString(String input) { - TextDocument intext = TextDocument.readOnlyString(Chars.wrap(input), LanguageRegistry.getDefaultLanguage().getDefaultVersion()); + TextDocument intext = TextDocument.readOnlyString(Chars.wrap(input), DummyLanguageModule.getInstance().getDefaultVersion()); return new JavaEscapeTranslator(intext).translateDocument(); }