1 /***
2 * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3 */
4 package net.sourceforge.pmd.cpd;
5
6 import net.sourceforge.pmd.cpd.cppast.CPPParserTokenManager;
7 import net.sourceforge.pmd.cpd.cppast.Token;
8 import net.sourceforge.pmd.cpd.cppast.TokenMgrError;
9 import net.sourceforge.pmd.cpd.cppast.SimpleCharStream;
10
11 import java.io.StringReader;
12 import java.util.ArrayList;
13 import java.util.List;
14
15 public class CPPTokenizer implements Tokenizer {
16 protected String EOL = System.getProperty("line.separator", "\n");
17
18 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
19 StringBuffer sb = sourceCode.getCodeBuffer();
20 try {
21 CPPParserTokenManager tokenManager = new CPPParserTokenManager(new SimpleCharStream(new StringReader(sb.toString())));
22 Token currToken = tokenManager.getNextToken();
23 while (currToken.image.length() > 0) {
24 tokenEntries.add(new TokenEntry(currToken.image, sourceCode.getFileName(), currToken.beginLine));
25 currToken = tokenManager.getNextToken();
26 }
27 tokenEntries.add(TokenEntry.getEOF());
28 System.out.println("Added " + sourceCode.getFileName());
29 } catch (TokenMgrError err) {
30 err.printStackTrace();
31 System.out.println("Skipping " + sourceCode.getFileName() + " due to parse error");
32 List emptyCode = new ArrayList();
33 emptyCode.add("");
34 tokenEntries.add(TokenEntry.getEOF());
35 }
36 }
37 }