phoedos/pmd
1
1
Fork 1
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use it to replace RubyTokenizer

Browse Source
This commit is contained in:
Clément Fournier
2020-09-01 17:13:02 +02:00
parent b27ab97684
commit bc0d9c781d
6 changed files with 134 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pmd-core/src/main/java/net/sourceforge/pmd/cpd/AnyTokenizer.java
View File

@ -76,8 +76,8 @@ public class AnyTokenizer implements Tokenizer {
continue;
}
int bcol = 1 + matcher.start() - lastLineStart;
int ecol = 1 + StringUtil.columnNumberAt(image, image.length());
int bcol = 1 + matcher.start() - lastLineStart; // + 1 because columns are 1 based
int ecol = StringUtil.columnNumberAt(image, image.length()); // this already outputs a 1-based column
tokenEntries.add(new TokenEntry(image, sourceCode.getFileName(), lineNo, bcol, ecol));
}
} finally {

2
pmd-ruby/src/main/java/net/sourceforge/pmd/cpd/RubyLanguage.java
View File

@ -15,6 +15,6 @@ public class RubyLanguage extends AbstractLanguage {
* Creates a new Ruby Language instance.
*/
public RubyLanguage() {
super("Ruby", "ruby", new RubyTokenizer(), ".rb", ".cgi", ".class");
super("Ruby", "ruby", new AnyTokenizer("#"), ".rb", ".cgi", ".class");
}
}

2
pmd-ruby/src/main/java/net/sourceforge/pmd/cpd/RubyTokenizer.java
View File

@ -10,7 +10,9 @@ import java.util.ArrayList;
* Tokenizer for Ruby.
*
* @author Zev Blut zb@ubit.com
* @deprecated Was replaced by an {@link AnyTokenizer}. Use {@link RubyLanguage#getTokenizer()} anyway
*/
@Deprecated
public class RubyTokenizer extends AbstractTokenizer {
/**

2
pmd-ruby/src/test/java/net/sourceforge/pmd/cpd/RubyTokenizerTest.java
View File

@ -23,7 +23,7 @@ public class RubyTokenizerTest extends CpdTextComparisonTest {
@Override
public Tokenizer newTokenizer(Properties properties) {
return new RubyTokenizer();
return new RubyLanguage().getTokenizer();
}

95
pmd-ruby/src/test/resources/net/sourceforge/pmd/lang/ruby/cpd/testdata/server.txt vendored
View File

@ -1,44 +1,79 @@
[Image] or [Truncated image[ Bcol Ecol
L1
[require] 1 7
["socket"] 9 16
[require] 1 8
["socket"] 9 9
L3
[gs] 1 2
[=] 4 4
[tcpserver.open] 7 20
[0] 22 22
[gs] 1 3
[=] 4 2
[TCPServer] 7 10
[.] 16 2
[open] 17 5
[(] 21 2
[0] 22 2
[)] 23 2
L4
[addr] 1 4
[=] 7 7
[gs.addr] 9 15
[addr] 1 5
[=] 7 2
[gs] 9 3
[.] 11 2
[addr] 12 5
L5
[addr.shift] 1 10
[addr] 1 5
[.] 5 2
[shift] 6 6
L7
[true] 7 10
[while] 1 6
[true] 7 5
L8
[ns] 3 4
[=] 6 6
[gs.accept] 8 16
[ns] 3 3
[=] 6 2
[gs] 8 3
[.] 10 2
[accept] 11 7
L9
[print] 3 7
[ns] 9 10
[" is accepted"] 13 26
[print] 3 6
[(] 8 2
[ns] 9 3
[,] 11 2
[" is accepted"] 13 15
[)] 27 2
L10
[thread.start] 3 14
[Thread] 3 7
[.] 9 2
[start] 10 6
[do] 16 3
L11
[s] 5 5
[=] 7 7
[ns] 9 10
[ # save to dynamic variable] 33 58
[s] 5 2
[=] 7 2
[ns] 9 3
L12
[s.gets] 11 16
[while] 5 6
[s] 11 2
[.] 12 2
[gets] 13 5
L13
[s.write] 7 13
[$_] 15 16
[s] 7 2
[.] 8 2
[write] 9 6
[(] 14 2
[$] 15 2
[_] 16 2
[)] 17 2
L14
[end] 5 4
L15
[print] 5 9
[s] 11 11
[" is gone [ 14 36
L19
[s.close] 5 11
[print] 5 6
[(] 10 2
[s] 11 2
[,] 12 2
[" is \n gone\n [ 14 37
[)] 103 2
L16
[s] 5 2
[.] 6 2
[close] 7 6
L17
[end] 3 4
L18
[end] 1 4
EOF

91
pmd-ruby/src/test/resources/net/sourceforge/pmd/lang/ruby/cpd/testdata/tabWidth.txt vendored
View File

@ -1,44 +1,79 @@
[Image] or [Truncated image[ Bcol Ecol
L1
[require] 1 7
["socket"] 9 16
[require] 1 8
["socket"] 9 9
L3
[gs] 1 2
[=] 4 4
[tcpserver.open] 7 20
[0] 22 22
[gs] 1 3
[=] 4 2
[TCPServer] 7 10
[.] 16 2
[open] 17 5
[(] 21 2
[0] 22 2
[)] 23 2
L4
[addr] 1 4
[=] 7 7
[gs.addr] 9 15
[addr] 1 5
[=] 7 2
[gs] 9 3
[.] 11 2
[addr] 12 5
L5
[addr.shift] 1 10
[addr] 1 5
[.] 5 2
[shift] 6 6
L7
[true] 7 10
[while] 1 6
[true] 7 5
L8
[ns] 2 3
[=] 5 5
[gs.accept] 7 15
[=] 5 2
[gs] 7 3
[.] 9 2
[accept] 10 7
L9
[print] 2 6
[ns] 8 9
[" is accepted"] 12 25
[(] 7 2
[ns] 8 3
[,] 10 2
[" is accepted"] 12 15
[)] 26 2
L10
[thread.start] 2 13
[Thread] 2 7
[.] 8 2
[start] 9 6
[do] 15 3
L11
[s] 3 3
[=] 5 5
[ns] 7 8
[ # save to dynamic variable] 31 56
[s] 3 2
[=] 5 2
[ns] 7 3
L12
[s.gets] 9 14
[while] 3 6
[s] 9 2
[.] 10 2
[gets] 11 5
L13
[s.write] 4 10
[$_] 12 13
[s] 4 2
[.] 5 2
[write] 6 6
[(] 11 2
[$] 12 2
[_] 13 2
[)] 14 2
L14
[end] 3 4
L15
[print] 3 7
[s] 9 9
[" is \t\t gone\t\t [ 12 34
L19
[s.close] 3 9
[print] 3 6
[(] 8 2
[s] 9 2
[,] 10 2
[" is \n\t\t gone\n\t\t [ 12 35
[)] 95 2
L16
[s] 3 2
[.] 4 2
[close] 5 6
L17
[end] 2 4
L18
[end] 1 4
EOF