Fix tests

This commit is contained in:
Clément Fournier
2020-09-01 17:56:12 +02:00
parent 251a217a59
commit dd440eb14a
3 changed files with 11 additions and 15 deletions
pmd-core/src
main/java/net/sourceforge/pmd/cpd
test/java/net/sourceforge/pmd/cpd

@ -7,8 +7,6 @@ package net.sourceforge.pmd.cpd;
import java.util.List;
import java.util.Locale;
import net.sourceforge.pmd.annotation.DeprecatedUntil700;
/**
*
* @author Zev Blut zb@ubit.com
@ -17,7 +15,6 @@ import net.sourceforge.pmd.annotation.DeprecatedUntil700;
* @deprecated Use an {@link AnyTokenizer} instead, it's basically as powerful.
*/
@Deprecated
@DeprecatedUntil700
public abstract class AbstractTokenizer implements Tokenizer {
// FIXME depending on subclasses to assign local vars is rather fragile -

@ -18,7 +18,7 @@ import net.sourceforge.pmd.util.StringUtil;
* Higher-quality lexers should be implemented with a lexer generator.
*
* <p>In PMD 7, this replaces AbstractTokenizer, which provided nearly
* no more functionality and whose API was hard to update.
* no more functionality.
*/
public class AnyTokenizer implements Tokenizer {
@ -61,7 +61,7 @@ public class AnyTokenizer implements Tokenizer {
@Override
public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
StringBuilder text = sourceCode.getCodeBuffer();
CharSequence text = sourceCode.getCodeBuffer();
Matcher matcher = pattern.matcher(text);
int lineNo = 1;
int lastLineStart = 0;

@ -7,14 +7,11 @@ package net.sourceforge.pmd.cpd;
import static net.sourceforge.pmd.util.CollectionUtil.listOf;
import static org.junit.Assert.assertEquals;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import org.checkerframework.checker.nullness.qual.NonNull;
import org.junit.Test;
import net.sourceforge.pmd.internal.util.IteratorUtil;
public class AnyTokenizerTest {
@Test
@ -33,13 +30,13 @@ public class AnyTokenizerTest {
public void testMultilineString() {
AnyTokenizer tokenizer = new AnyTokenizer("//");
Tokens tokens = compareResult(tokenizer, "a = \"oo\n\";", listOf("a", "=", "\"oo\n\"", ";", "EOF"));
TokenEntry string = IteratorUtil.getNth(tokens.iterator(), 2);
TokenEntry string = tokens.getTokens().get(2);
assertEquals("\"oo\n\"", getTokenImage(string));
assertEquals(1, string.getBeginLine());
assertEquals(5, string.getBeginColumn());
assertEquals(2, string.getEndColumn()); // ends on line 2
TokenEntry semi = IteratorUtil.getNth(tokens.iterator(), 3);
TokenEntry semi = tokens.getTokens().get(3);
assertEquals(";", getTokenImage(semi));
assertEquals(2, semi.getBeginLine());
assertEquals(2, semi.getBeginColumn());
@ -50,15 +47,17 @@ public class AnyTokenizerTest {
SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(source));
Tokens tokens = new Tokens();
tokenizer.tokenize(code, tokens);
List<String> tokenStrings = tokens.getTokens().stream()
.map(this::getTokenImage)
.collect(Collectors.toList());
List<String> tokenStrings = new ArrayList<>();
for (TokenEntry token : tokens.getTokens()) {
tokenStrings.add(getTokenImage(token));
}
assertEquals(expectedImages, tokenStrings);
return tokens;
}
private @NonNull String getTokenImage(TokenEntry t) {
private String getTokenImage(TokenEntry t) {
return t.toString();
}