Skip to content

Commit

Permalink
Improved code quality and fixed typos.
Browse files Browse the repository at this point in the history
  • Loading branch information
TwoOfTwelve committed Aug 26, 2024
1 parent 4f512d5 commit 0923193
Show file tree
Hide file tree
Showing 7 changed files with 38 additions and 36 deletions.
8 changes: 4 additions & 4 deletions docs/4.-Adding-New-Languages.md
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ protected File getTestFileLocation() {

## Testing token positions

The precise position of a token can be relevant for the visualization in the report viewer. To make sure the token positions are extracted correctly language modules should include some test for that.
The precise position of a token can be relevant for the visualization in the report viewer. To make sure the token positions are extracted correctly language modules should include some tests for that.

Writing such tests can be done using a specific syntax in the test sources directly.
Such a file can look like this:
Expand All @@ -527,11 +527,11 @@ Every line that is prefixed with '>' will be interpreted as a line of test sourc

Every line starting with '$' contains information about one expected token. The token is expected in the first source line above this one.
The '|' marks the column the token should be in. It is followed by one space, then the name of the token (The name of the enum constant).
Finally separated with another space is the length of the token.
Finally, separated by another space is the length of the token.
A single file may contain any number of tokens.
The test will verify that at least one token with those exact properties exist.
The test will verify that at least one token with those exact properties exists.

These test files have to be added in the TestDataCollector. Put all test files in a single directory and specify it though collector.addTokenPositionTests("<directory>").
These test files have to be added in the TestDataCollector. Put all test files in a single directory and specify it through collector.addTokenPositionTests("<directory>").
If the directory is in the default location for test files, a relative path is enough, otherwise a full path has to be specified.


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -215,16 +215,18 @@ final void testTokenPositions(TokenPositionTestData testData) throws ParsingExce
TokenType expectedType = this.languageTokens.stream().filter(type -> type.toString().equals(expectedToken.typeName())).findFirst()
.orElseThrow(() -> new IOException(String.format("The token type %s does not exist.", expectedToken.typeName())));

if (extractedTokens.stream().noneMatch(token -> token.getType() == expectedType && token.getLine() == expectedToken.line()
&& token.getColumn() == expectedToken.col() && token.getLength() == expectedToken.length())) {
if (extractedTokens.stream().noneMatch(token -> token.getType() == expectedType && token.getLine() == expectedToken.lineNumber()
&& token.getColumn() == expectedToken.columnNumber() && token.getLength() == expectedToken.length())) {
failedTokens.add(expectedToken);
}
}

if (!failedTokens.isEmpty()) {
String failDescriptors = String.join(System.lineSeparator(), failedTokens.stream()
.map(token -> token.typeName() + " at (" + token.line() + ":" + token.col() + ") with length " + token.length()).toList());
fail("Some tokens weren't extracted with the correct properties:" + System.lineSeparator() + failDescriptors);
String failureDescriptors = String.join(System.lineSeparator(),
failedTokens.stream().map(
token -> token.typeName() + " at (" + token.lineNumber() + ":" + token.columnNumber() + ") with length " + token.length())
.toList());
fail("Some tokens weren't extracted with the correct properties:" + System.lineSeparator() + failureDescriptors);
}
}

Expand Down Expand Up @@ -290,8 +292,8 @@ final void collectTestData() {
}

@AfterAll
final void releaseTmpFiles() {
TmpFileHolder.deleteTmpFiles();
final void deleteTemporaryFiles() {
TemporaryFileHolder.deleteTemporaryFiles();
}

private List<Token> parseTokens(TestData source) throws ParsingException, IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
/**
* Stores all temporary files that are created for a {@link LanguageModuleTest} and provides the option to delete them
*/
public class TmpFileHolder {
public static List<File> tmpFiles = new ArrayList<>();
public class TemporaryFileHolder {
public static List<File> temporaryFiles = new ArrayList<>();

/**
* Deletes all temporary files that have been created up to this point
*/
public static void deleteTmpFiles() {
tmpFiles.forEach(File::delete);
tmpFiles.clear();
public static void deleteTemporaryFiles() {
temporaryFiles.forEach(File::delete);
temporaryFiles.clear();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
import de.jplag.Language;
import de.jplag.ParsingException;
import de.jplag.Token;
import de.jplag.testutils.TmpFileHolder;
import de.jplag.testutils.TemporaryFileHolder;
import de.jplag.util.FileUtils;

/**
Expand All @@ -26,7 +26,7 @@ public List<Token> parseTokens(Language language) throws ParsingException, IOExc
File file = File.createTempFile("testSource", language.suffixes()[0]);
FileUtils.write(file, this.testData);
List<Token> tokens = language.parse(Collections.singleton(file));
TmpFileHolder.tmpFiles.add(file);
TemporaryFileHolder.temporaryFiles.add(file);
return tokens;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -86,18 +86,18 @@ public TestDataContext inlineSource(String... sources) {
* @throws IOException If the files cannot be read
*/
public TestDataContext addTokenPositionTests(String directoryName) {
File dir = new File(this.testFileLocation, directoryName);
Set<TestData> allData = new HashSet<>();
for (File file : Objects.requireNonNull(dir.listFiles())) {
File directory = new File(this.testFileLocation, directoryName);
Set<TestData> allTestsInDirectory = new HashSet<>();
for (File file : Objects.requireNonNull(directory.listFiles())) {
try {
TokenPositionTestData data = new TokenPositionTestData(file);
allData.add(data);
allTestsInDirectory.add(data);
this.tokenPositionTestData.add(data);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
return new TestDataContext(allData);
return new TestDataContext(allTestsInDirectory);
}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,12 @@
import de.jplag.Language;
import de.jplag.ParsingException;
import de.jplag.Token;
import de.jplag.testutils.TmpFileHolder;
import de.jplag.testutils.TemporaryFileHolder;
import de.jplag.util.FileUtils;

/**
* Test sources with token information Reads token position test specifications form a file and provides the token
* information for tests. The sources cna be used as regular test sources.
* information for tests. The sources can be used as regular test sources.
*/
public class TokenPositionTestData implements TestData {
private final List<String> sourceLines;
Expand Down Expand Up @@ -44,12 +44,12 @@ private void readFile(File testFile) throws IOException {
}

if (sourceLine.charAt(0) == '$') {
int col = sourceLine.indexOf('|');
String[] parts = sourceLine.split(" ", 0);
int column = sourceLine.indexOf('|');
String[] tokenDescriptionParts = sourceLine.split(" ", 0);

String typeName = parts[parts.length - 2];
int length = Integer.parseInt(parts[parts.length - 1]);
this.expectedTokens.add(new TokenData(typeName, currentLine, col, length));
String typeName = tokenDescriptionParts[tokenDescriptionParts.length - 2];
int length = Integer.parseInt(tokenDescriptionParts[tokenDescriptionParts.length - 1]);
this.expectedTokens.add(new TokenData(typeName, currentLine, column, length));
}
}
}
Expand All @@ -59,7 +59,7 @@ public List<Token> parseTokens(Language language) throws ParsingException, IOExc
File file = File.createTempFile("testSource", language.suffixes()[0]);
FileUtils.write(file, String.join(System.lineSeparator(), sourceLines));
List<Token> tokens = language.parse(Collections.singleton(file));
TmpFileHolder.tmpFiles.add(file);
TemporaryFileHolder.temporaryFiles.add(file);
return tokens;
}

Expand All @@ -83,10 +83,10 @@ public List<TokenData> getExpectedTokens() {
/**
* Information about a single token
* @param typeName The name of the token type
* @param line The line the token is in
* @param col The column the token is in
* @param lineNumber The line the token is in (1 based)
* @param columnNumber The column the token is in (1 based)
* @param length The length of the token
*/
public record TokenData(String typeName, int line, int col, int length) {
public record TokenData(String typeName, int lineNumber, int columnNumber, int length) {
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -83,8 +83,8 @@ private void parseFile(File file) throws ParsingException {
/**
* Adds a new {@link Token} to the current token list.
* @param type the type of the new {@link Token}
* @param line the line of the Token in the current file
* @param start the start column of the Token in the line
* @param line the lineNumber of the Token in the current file
* @param start the start column of the Token in the lineNumber
* @param length the length of the Token
*/
/* package-private */ void addToken(TokenType type, int line, int start, int length) {
Expand Down

0 comments on commit 0923193

Please sign in to comment.