Clover Coverage Report
Coverage timestamp: Fri May 9 2008 10:54:27 EST
Overview   Package   File
    FRAMES   NO FRAMES     
../../../../../img/srcFileCovDistChart4.png 81% of files have more coverage
114   206   46   7.6
42   166   0.4   15
15     3.07  
1    
 
  StandardTokenizer       Line # 22 114 46 36.8% 0.36842105
 
  (32)
 
1    /* Generated By:JavaCC: Do not edit this line. StandardTokenizer.java */
2    package org.apache.lucene.analysis.standard;
3   
4    import java.io.*;
5   
6    /** A grammar-based tokenizer constructed with JavaCC.
7    *
8    * <p> This should be a good tokenizer for most European-language documents:
9    *
10    * <ul>
11    * <li>Splits words at punctuation characters, removing punctuation. However, a
12    * dot that's not followed by whitespace is considered part of a token.
13    * <li>Splits words at hyphens, unless there's a number in the token, in which case
14    * the whole token is interpreted as a product number and is not split.
15    * <li>Recognizes email addresses and internet hostnames as one token.
16    * </ul>
17    *
18    * <p>Many applications have specific tokenizer needs. If this tokenizer does
19    * not suit your application, please consider copying this source code
20    * directory to your project and maintaining your own grammar-based tokenizer.
21    */
 
22    public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer implements StandardTokenizerConstants {
23   
24    /** Constructs a tokenizer for this Reader. */
 
25  6567 toggle public StandardTokenizer(Reader reader) {
26  6567 this(new FastCharStream(reader));
27  6567 this.input = reader;
28    }
29   
30    /** Returns the next token in the stream, or null at EOS.
31    * <p>The returned token's type is set to an element of {@link
32    * StandardTokenizerConstants#tokenImage}.
33    */
 
34  13797 toggle final public org.apache.lucene.analysis.Token next() throws ParseException, IOException {
35  13797 Token token = null;
36  13797 switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
37  7207 case ALPHANUM:
38  7207 token = jj_consume_token(ALPHANUM);
39  7207 break;
40  10 case APOSTROPHE:
41  10 token = jj_consume_token(APOSTROPHE);
42  10 break;
43  1 case ACRONYM:
44  1 token = jj_consume_token(ACRONYM);
45  1 break;
46  2 case COMPANY:
47  2 token = jj_consume_token(COMPANY);
48  2 break;
49  3 case EMAIL:
50  3 token = jj_consume_token(EMAIL);
51  3 break;
52  4 case HOST:
53  4 token = jj_consume_token(HOST);
54  4 break;
55  3 case NUM:
56  3 token = jj_consume_token(NUM);
57  3 break;
58  0 case CJ:
59  0 token = jj_consume_token(CJ);
60  0 break;
61  6567 case 0:
62  6567 token = jj_consume_token(0);
63  6567 break;
64  0 default:
65  0 jj_la1[0] = jj_gen;
66  0 jj_consume_token(-1);
67  0 throw new ParseException();
68    }
69  13797 if (token.kind == EOF) {
70  6567 {if (true) return null;}
71    } else {
72  7230 {if (true) return
73    new org.apache.lucene.analysis.Token(token.image,
74    token.beginColumn,token.endColumn,
75    tokenImage[token.kind]);}