☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆☆AnalyzerTool分词工具.非常实用!

AnalyzerTool分词工具.非常实用!

可以查看某串字符最终被分割成什么样子,这样便于查询时深刻明白为什么有的查不到有的却能查到.

package test.main;

import java.io.IOException;

import java.io.StringReader;

import org.apache.lucene.analysis.Analyzer;

import org.apache.lucene.analysis.TokenStream;

import org.apache.lucene.analysis.core.SimpleAnalyzer;

import org.apache.lucene.analysis.core.StopAnalyzer;

import org.apache.lucene.analysis.core.WhitespaceAnalyzer;

import org.apache.lucene.analysis.standard.StandardAnalyzer;

import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

import org.apache.lucene.util.Version;

/**

 * 展示分词后的效果

 */

public class AnalyzerTool {

    /**

     * 打印分词后的信息

     *

     * @param str

     *            待分词的字符串

     * @param analyzer

     *            分词器

     */

    public static void displayToken(String str, Analyzer analyzer) {

        TokenStream stream = null;

        try {

            // 将一个字符串创建成Token流

            stream = analyzer.tokenStream("content", new StringReader(str));

            CharTermAttribute cta = stream.addAttribute(CharTermAttribute.class);

            stream.reset();// 一定要重置,不然老报错

            while (stream.incrementToken()) {

                System.out.print("【" + cta + "】");

            }

            System.out.println();

        } catch (IOException e) {

            e.printStackTrace();

        } finally {

            try {

                if (stream != null) {

                    stream.end();

                    stream.close();

                }

            } catch (Exception e) {

                // TODO Auto-generated catch block

                e.printStackTrace();

            }

        }

    }

    public static void main(String[] args) {

        Analyzer aly1 = new StandardAnalyzer(Version.LUCENE_40);

        Analyzer aly2 = new StopAnalyzer(Version.LUCENE_40);

        Analyzer aly3 = new SimpleAnalyzer(Version.LUCENE_40);

        Analyzer aly4 = new WhitespaceAnalyzer(Version.LUCENE_40);

        String str = "-LT1TT132#########LJRT1326#########LJRT1226#########)";

        // LT1TT132#########LJRT1326#########LJRT1226#########

        // LA939VRG###AJA###LA939VRG###WSJ###

        // LA99HRD3###SYC###

        // LZ1B22EE#########

        AnalyzerTool.displayToken(str, aly1);

        AnalyzerTool.displayToken(str, aly2);

        AnalyzerTool.displayToken(str, aly3);

        AnalyzerTool.displayToken(str, aly4);

    }

}