LexicalScanner.java

Go to the documentation of this file.

package aud.util;
 
import java.util.regex.*;
 
public class LexicalScanner {
 
  public final static int END_OF_INPUT = -1;
  public final static int NO_MATCH     = -2;
 
  public static class Rule {
    public Rule(int id, Pattern pattern) {
      id_=id;
      pattern_=pattern;
    }
 
    public Rule(int id, String pattern) {
      this(id,Pattern.compile(pattern));
    }
 
    int id_;
    Pattern pattern_;
  }
 
  String input_ = null;
  String text_  = null;
  int    id_    = NO_MATCH;
  Rule[] rules_ = null;
 
  public static final Pattern P_WHITESPACE = Pattern.compile("\\s+");
  public static final Pattern P_IDENTIFIER =
    Pattern.compile("[_a-zA-Z]?(\\w|_)+");
  public static final Pattern P_FLOAT =
    Pattern.compile("[-+]?[0-9]*\\.?[0-9]+([eE][-+]?[0-9]+)?");
 
  public LexicalScanner(Rule[] rules,String input) {
    rules_=rules;
    input_=input;
  }
 
  protected void eatWhiteSpace() {
    if (!endOfInput()) {
      Matcher m=P_WHITESPACE.matcher(input_);
      if (m.lookingAt()) {
        input_=input_.substring(m.end(),input_.length());
      }
    }
  }
 
  public void setInput(String input) {
    input_=input;
  }
  public String matchedText() { return text_; }
  public int matchedTokenId() { return id_; }
  public String remainder() { return input_; }
  public boolean endOfInput() {
    return input_==null || input_.length()==0;
  }
 
  protected boolean match(Pattern p) {
    text_=null;
    eatWhiteSpace();
 
    if (endOfInput())
      return false;
 
   Matcher m=p.matcher(input_);
    if (!m.lookingAt())
      return false;
 
    int n=m.end();
    text_=input_.substring(0,n);
    input_=input_.substring(m.end(),input_.length());
 
    return true;
  }
  protected int next(Rule[] rules) {
    eatWhiteSpace();
 
    if (endOfInput())  return id_=END_OF_INPUT;
    if (rules_==null)  return id_=NO_MATCH;
 
    for (Rule rule : rules) {
      if (match(rule.pattern_))
        return id_=rule.id_;
    }
    return id_=NO_MATCH;
  }
 
  public int next() { return next(rules_);  }
 
 
  public static void main(String[] args) {
 
    Rule[] rules={
      new Rule(1,"[0-9]*\\.?[0-9]+"),
      new Rule(2,"[a-z]+")
    };
 
    LexicalScanner s=new LexicalScanner
      (rules,args.length==0 ? "  12.3a 12 bcd 34 " : args[0]);
 
    System.out.println("input = '"+s.remainder()+"'");
 
    while (s.next()!=END_OF_INPUT) {
      if (s.matchedTokenId()==NO_MATCH) {
        System.out.println("syntax error near '"+s.remainder()+"'");
        break;
      }
      System.out.println("next token id = "+s.matchedTokenId());
      System.out.println("matched text = '"+s.matchedText()+"'");
      System.out.println("remaining input = '"+s.remainder()+"'");
    }
  }
}