From 817c3d112734037b969c72edbe43e11e19e96c9e Mon Sep 17 00:00:00 2001 From: Andreas Dangel Date: Fri, 17 Jan 2020 14:47:23 +0100 Subject: [PATCH] CPD: Remove deprecations in Mark / TokenEntry * Provide these values for more languages --- .../sourceforge/pmd/cpd/ApexTokenizer.java | 3 ++- .../pmd/cpd/AbstractTokenizer.java | 3 +-- .../java/net/sourceforge/pmd/cpd/Mark.java | 12 +++++++++-- .../net/sourceforge/pmd/cpd/TokenEntry.java | 21 ++++++++++++++----- .../sourceforge/pmd/cpd/GroovyTokenizer.java | 4 +++- .../pmd/cpd/EcmascriptTokenizer.java | 3 ++- .../net/sourceforge/pmd/cpd/JSPTokenizer.java | 2 +- .../sourceforge/pmd/cpd/PLSQLTokenizer.java | 3 ++- .../sourceforge/pmd/cpd/ScalaTokenizer.java | 5 ++++- .../net/sourceforge/pmd/cpd/VfTokenizer.java | 2 +- 10 files changed, 42 insertions(+), 16 deletions(-) diff --git a/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java b/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java index c872489c02..bb5f9b6cf1 100644 --- a/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java +++ b/pmd-apex/src/main/java/net/sourceforge/pmd/cpd/ApexTokenizer.java @@ -55,7 +55,8 @@ public class ApexTokenizer implements Tokenizer { if (!caseSensitive) { tokenText = tokenText.toLowerCase(Locale.ROOT); } - TokenEntry tokenEntry = new TokenEntry(tokenText, sourceCode.getFileName(), token.getLine()); + TokenEntry tokenEntry = new TokenEntry(tokenText, sourceCode.getFileName(), token.getLine(), + token.getCharPositionInLine(), token.getCharPositionInLine() + tokenText.length()); tokenEntries.add(tokenEntry); } token = lexer.nextToken(); diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java index 7309fd2f4c..87c5acb924 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/AbstractTokenizer.java @@ -58,8 +58,7 @@ public abstract class AbstractTokenizer implements Tokenizer { // if ( CPD.debugEnable ) { // System.out.println("Token added:" + token.toString()); // } - tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber + 1)); - + tokenEntries.add(new TokenEntry(token.toString(), tokens.getFileName(), lineNumber + 1, loc - token.length(), loc - 1)); } } } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java index 041ac112d3..a1b88be336 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/Mark.java @@ -26,7 +26,11 @@ public class Mark implements Comparable { return this.token.getBeginLine(); } - @Deprecated + /** + * The column number where this duplication begins. + * returns -1 if not available + * @return the begin column number + */ public int getBeginColumn() { return this.token.getBeginColumn(); // TODO Java 1.8 make optional } @@ -35,7 +39,11 @@ public class Mark implements Comparable { return getBeginLine() + getLineCount() - 1; } - @Deprecated + /** + * The column number where this duplication ends. + * returns -1 if not available + * @return the end column number + */ public int getEndColumn() { return this.endToken == null ? -1 : this.endToken.getEndColumn(); // TODO Java 1.8 make optional } diff --git a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java index 15074fc1f5..7246dc6acf 100644 --- a/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java +++ b/pmd-core/src/main/java/net/sourceforge/pmd/cpd/TokenEntry.java @@ -15,9 +15,9 @@ public class TokenEntry implements Comparable { public static final TokenEntry EOF = new TokenEntry(); private String tokenSrcID; - private int beginLine; - private int beginColumn; - private int endColumn; + private final int beginLine; + private final int beginColumn; + private final int endColumn; private int index; private int identifier; private int hashCode; @@ -38,6 +38,9 @@ public class TokenEntry implements Comparable { private TokenEntry() { this.identifier = 0; this.tokenSrcID = "EOFMarker"; + this.beginLine = -1; + this.beginColumn = -1; + this.endColumn = -1; } /** @@ -109,12 +112,20 @@ public class TokenEntry implements Comparable { return beginLine; } - @Deprecated + /** + * The column number where this token begins. + * returns -1 if not available + * @return the begin column number + */ public int getBeginColumn() { return beginColumn; // TODO Java 1.8 make optional } - @Deprecated + /** + * The column number where this token ends. + * returns -1 if not available + * @return the end column number + */ public int getEndColumn() { return endColumn; // TODO Java 1.8 make optional } diff --git a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java index 470c3b1cad..7470ad8ee9 100644 --- a/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java +++ b/pmd-groovy/src/main/java/net/sourceforge/pmd/cpd/GroovyTokenizer.java @@ -30,7 +30,9 @@ public class GroovyTokenizer implements Tokenizer { Token token = tokenStream.nextToken(); while (token.getType() != Token.EOF_TYPE) { - TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine()); + String tokenText = token.getText(); + TokenEntry tokenEntry = new TokenEntry(tokenText, sourceCode.getFileName(), token.getLine(), + token.getColumn(), tokenText.length()); tokenEntries.add(tokenEntry); token = tokenStream.nextToken(); diff --git a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java index 64c7914912..ebfb3e4ce8 100644 --- a/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java +++ b/pmd-javascript/src/main/java/net/sourceforge/pmd/cpd/EcmascriptTokenizer.java @@ -27,7 +27,8 @@ public class EcmascriptTokenizer extends JavaCCTokenizer { @Override protected TokenEntry processToken(Tokens tokenEntries, GenericToken currentToken, String filename) { - return new TokenEntry(getTokenImage(currentToken), filename, currentToken.getBeginLine()); + return new TokenEntry(getTokenImage(currentToken), filename, currentToken.getBeginLine(), + currentToken.getBeginColumn(), currentToken.getEndColumn()); } private String getTokenImage(GenericToken token) { diff --git a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java index 48cfb26500..559f6b40a4 100644 --- a/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java +++ b/pmd-jsp/src/main/java/net/sourceforge/pmd/cpd/JSPTokenizer.java @@ -30,7 +30,7 @@ public class JSPTokenizer implements Tokenizer { while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), - currentToken.beginLine)); + currentToken.beginLine, currentToken.beginColumn, currentToken.endColumn)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) { diff --git a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java index b44122309b..40126ce624 100644 --- a/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java +++ b/pmd-plsql/src/main/java/net/sourceforge/pmd/cpd/PLSQLTokenizer.java @@ -68,7 +68,8 @@ public class PLSQLTokenizer extends JavaCCTokenizer { image = String.valueOf(plsqlToken.kind); } - return new TokenEntry(image, fileName, currentToken.getBeginLine()); + return new TokenEntry(image, fileName, currentToken.getBeginLine(), + currentToken.getBeginColumn(), currentToken.getEndColumn()); } @Override diff --git a/pmd-scala/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java b/pmd-scala/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java index 7706e48d31..94c48abdc2 100644 --- a/pmd-scala/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java +++ b/pmd-scala/src/main/java/net/sourceforge/pmd/cpd/ScalaTokenizer.java @@ -17,6 +17,7 @@ import net.sourceforge.pmd.lang.scala.ScalaLanguageModule; import scala.collection.Iterator; import scala.meta.Dialect; import scala.meta.inputs.Input; +import scala.meta.inputs.Position; import scala.meta.internal.tokenizers.ScalametaTokenizer; import scala.meta.tokens.Token; @@ -74,7 +75,9 @@ public class ScalaTokenizer implements Tokenizer { Token token; while ((token = filter.getNextToken()) != null) { String tokenText = token.text() != null ? token.text() : token.name(); - TokenEntry cpdToken = new TokenEntry(tokenText, filename, token.pos().startLine()); + Position tokenPosition = token.pos(); + TokenEntry cpdToken = new TokenEntry(tokenText, filename, tokenPosition.startLine(), + tokenPosition.startColumn(), tokenPosition.endColumn()); tokenEntries.add(cpdToken); } } finally { diff --git a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java index 9ddce331de..13febc7b90 100644 --- a/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java +++ b/pmd-visualforce/src/main/java/net/sourceforge/pmd/cpd/VfTokenizer.java @@ -34,7 +34,7 @@ public class VfTokenizer implements Tokenizer { while (currentToken.image.length() > 0) { tokenEntries.add(new TokenEntry(String.valueOf(currentToken.kind), sourceCode.getFileName(), - currentToken.beginLine)); + currentToken.beginLine, currentToken.beginColumn, currentToken.endColumn)); currentToken = (Token) tokenMgr.getNextToken(); } } catch (IOException e) {