下载IK源码
https://github.com/medcl/elasticsearch-analysis-ik/tree/v5.2.0
选择你对应ik的版本(ps:版本最好一致)
http://localhost:9200/?pretty查看es版本 我的是6.5.1
修改源码
1.创建一个ext包同时增加3个类文件
DBHelper
package org.wltea.analyzer.ext; import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.logging.Loggers; import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.HashMap;
import java.util.Map; public class DBHelper {
Logger logger= Loggers.getLogger(DBHelper.class); public static String url = null;
public static String dbUser = null;
public static String dbPwd = null;
public static String dbTable = null;
private Connection conn;
public static Map<String, Date> lastImportTimeMap = new HashMap<String, Date>(); static{
try {
Class.forName("com.mysql.jdbc.Driver");// 加载Mysql数据驱动
} catch (Exception e) {
e.printStackTrace();
}
}
private Connection getConn() throws Exception {
try {
conn = DriverManager.getConnection(url, dbUser, dbPwd);// 创建数据连接
} catch (Exception e) {
logger.warn("异常了");
e.printStackTrace();
}
return conn;
} /**
* 从数据库获得分词信息
* @param key 字段名
* @param type 分词类型 0扩展分词 1停分词
* @param delete 是否有效 0有效 1无效
* @param flag 是否每次加载最新的
* @param synonyStr
* @return
* @throws Exception
*/
public String getKey(String key, Integer type,boolean delete,boolean flag,String synonyStr) throws Exception { conn = getConn();
StringBuilder data = new StringBuilder();
PreparedStatement ps = null;
ResultSet rs = null;
try {
StringBuilder sql = new StringBuilder("select * from " + dbTable + " where 1=1");
//lastImportTime 最新更新时间
Date lastImportTime = DBHelper.lastImportTimeMap.get(key);
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
if (lastImportTime != null && flag) {
sql.append(" and update_time > '" + sdf.format(lastImportTime) + "'");
}
sql.append(" and " + key + " !=''");
if(type!=null){
sql.append("and word_type="+type);
}
if(delete){
sql.append(" and delete_type="+1);
}else{
sql.append(" and delete_type="+0);
}
lastImportTime = new Date();
lastImportTimeMap.put(key,lastImportTime);
//如果打印出来的时间 和本地时间不一样，则要注意JVM时区是否和服务器系统时区一致
logger.warn("sql==={}",sql.toString());
System.out.print(conn);
ps = conn.prepareStatement(sql.toString());
rs = ps.executeQuery();
while (rs.next()) {
String value = rs.getString(key);
if (StringUtils.isNotBlank(value)) {
if (StringUtils.isNotBlank(synonyStr)) {
data.append(value + synonyStr);
} else {
data.append(value + ",");
}
} }
} catch (Exception e) {
e.printStackTrace();
} finally {
try {
if (ps != null) {
ps.close(); }
if (rs != null) {
rs.close();
} conn.close(); } catch (Exception e) {
e.printStackTrace();
}
}
return data.toString();
}
//测试
// public static void main(String[] args) throws Exception {
// DBHelper dbHelper=new DBHelper();
// String extWords=dbHelper.getKey("ext_word",true);
// List<String> extList = Arrays.asList(extWords.split(","));
// System.out.println(extList);
// // System.out.println(getKey("stopword"));
// // System.out.println(getKey("synonym"));
// LocalDate now=LocalDate.now();
//
// } }
DBRunnable
package org.wltea.analyzer.ext; import org.apache.logging.log4j.Logger;
import org.elasticsearch.common.logging.Loggers;
import org.wltea.analyzer.dic.Dictionary; import java.util.Arrays;
import java.util.List; public class DBRunnable implements Runnable {
Logger logger = Loggers.getLogger(DBRunnable.class);
private String wordField; public DBRunnable(String wordField) {
super();
this.wordField = wordField;
} @Override
public void run() {
logger.warn("开始加载词库========");
//获取词库
Dictionary dic = Dictionary.getSingleton();
DBHelper dbHelper = new DBHelper();
try {
String extWords = dbHelper.getKey(wordField, 0,false,true,",");
String stopWords = dbHelper.getKey(wordField, 1,false,true,",");
String extDelWords = dbHelper.getKey(wordField, 0,true,true,",");
String extStopWords = dbHelper.getKey(wordField, 1,true,true,",");
if(StringUtils.isNotBlank(extWords)){
List<String> extList = Arrays.asList(extWords.split(","));
//把扩展词加载到主词库中
dic.addWords(extList);
logger.warn("加载扩展词成功========");
logger.warn("extWords为==={}",extWords);
}
if(StringUtils.isNotBlank(stopWords)){
List<String> stopList = Arrays.asList(stopWords.split(","));
//把扩展词加载到主词库中
dic.addStopWords(stopList);
logger.warn("加载停用词成功========");
logger.warn("stopWords为==={}",stopWords);
}
//移除词库
if(StringUtils.isNotBlank(extDelWords)){
List<String> stopList = Arrays.asList(extDelWords.split(","));
//把扩展词加载到主词库中
dic.disableWords(stopList);
logger.warn("移除扩展词成功========");
logger.warn("extDelWords==={}",extDelWords);
}
if(StringUtils.isNotBlank(extStopWords)){
List<String> stopList = Arrays.asList(extStopWords.split(","));
//把扩展词加载到主词库中
dic.disableStopWords(stopList);
logger.warn("移除停用词成功========");
logger.warn("extStopWords==={}",extStopWords);
} } catch (Exception e) { logger.warn("加载扩展词失败========{}",e);
} } }
StringUtils
package org.wltea.analyzer.ext;
public class StringUtils {
/**
* 判断字符串是否为空 为空返回true 否则返回false
* @param str
* @return
*/
public static boolean isBlank(String str) {
int strLen;
if (str == null || (strLen = str.length()) == 0) {
return true;
}
for (int i = 0; i < strLen; i++) {
if ((Character.isWhitespace(str.charAt(i)) == false)) {
return false;
}
}
return true;
}
/**
* 判断字符串是否不为空 为空返回false 否则返回true
* @param str
* @return
*/
public static boolean isNotBlank(String str) {
return !StringUtils.isBlank(str);
}
}2.Dictionary增加几个方法
/**
* 批量加载新停用词条
*
* @param words
* Collection<String>词条列表
*/
public void addStopWords(Collection<String> words) {
if (words != null) {
for (String word : words) {
if (word != null) {
// 批量加载词条到主内存词典中
_StopWords.fillSegment(word.trim().toCharArray());
}
}
} }
/**
* 批量移除停用词条
*
* @param words
* Collection<String>词条列表
*/
public void disableStopWords(Collection<String> words) {
if (words != null) {
for (String word : words) {
if (word != null) {
// 批量加载词条到主内存词典中
_StopWords.disableSegment(word.trim().toCharArray());
}
}
} }
/**
* 读取jdbc配置初始化 定时更新数据库词组定时任务
*
* @throws IOException
*/
public void initReloadMysqlWordJob() throws IOException { logger.warn("============IKAnalyzer==============");
Path file = PathUtils.get(getDictRoot(), "jdbc.properties");
Properties prop = new Properties();
prop.load(new FileInputStream(file.toFile()));
logger.info("===========load jdbc.properties========");
for(Object key : prop.keySet()) {
logger.info("==========>>" + key + "=" + prop.getProperty(String.valueOf(key)));
}
boolean autoReloadDic=Boolean.valueOf(prop.getProperty("autoReloadDic"));
if(autoReloadDic){
String dbUser = prop.getProperty("dbUser");
String dbPwd = prop.getProperty("dbPwd");
//获取每隔多久从数据库更新信息 默认60S
Integer flushTime = Integer.valueOf(prop.getProperty("flushTime"));
String dbTable = prop.getProperty("dbTable","t_es_ik_dic");
DBHelper.dbTable=dbTable;
DBHelper.dbUser=dbUser;
DBHelper.dbPwd=dbPwd;
DBHelper.url=prop.getProperty("dbUrl");
String wordFieldName = prop.getProperty("wordFieldName");
ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor();
scheduledExecutorService.scheduleAtFixedRate(new DBRunnable(wordFieldName), 0, flushTime, TimeUnit.SECONDS);
}
}
4.在init方法启用job
public static synchronized Dictionary initial(Configuration cfg) {
if (singleton == null) {
synchronized (Dictionary.class) {
if (singleton == null) {
singleton = new Dictionary(cfg);
singleton.loadMainDict();
singleton.loadSurnameDict();
singleton.loadQuantifierDict();
singleton.loadSuffixDict();
singleton.loadPrepDict();
singleton.loadStopWordDict();
try {
singleton.initReloadMysqlWordJob();
} catch (IOException e) {
logger.error("动态加载mysql词组失败....");
e.printStackTrace();
}
if(cfg.isEnableRemoteDict()){
// 建立监控线程
for (String location : singleton.getRemoteExtDictionarys()) {
// 10 秒是初始延迟可以修改的 60是间隔时间 单位秒
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
}
for (String location : singleton.getRemoteExtStopWordDictionarys()) {
pool.scheduleAtFixedRate(new Monitor(location), 10, 60, TimeUnit.SECONDS);
}
}
return singleton;
}
}
}
return singleton;
}将ik安装导入es
1.打包
2.将zip文件移动到es的plugins文件夹
解压并重命名为ik
3.ik目录的config创建一个jdbc.properties文件
dbUrl=jdbc:mysql://ip/port #数据库连接
dbUser=user #数据库用户名
dbPwd=password #数据库密码
dbTable=md_es_ik_dic #词库表
wordFieldName=word #词组字段
flushTime=5 #刷新时间 (秒)
autoReloadDic=true #是否启用
4.创建数据库表
DROP TABLE IF EXISTS `md_es_ik_dic`;
CREATE TABLE `md_es_ik_dic` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '自增id',
`word` varchar(100) DEFAULT '' COMMENT '扩展分词',
`word_type` varchar(100) DEFAULT '' COMMENT '0:扩展分词 1:停用分词 ',
`delete_type` tinyint(4) DEFAULT '' COMMENT '0表示未删除，1表示删除',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '更新时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB AUTO_INCREMENT=16 DEFAULT CHARSET=utf8 COMMENT='词库维护表';
5.es lib增加一个mysql数据库驱动文件
6.启动es测试
get请求es:http://127.0.0.1:9200/_analyze
{
"analyzer":"ik_max_word",
"text":"我是一名小正太"
}分词结果
{
"tokens": [
{
"token": "我",
"start_offset": 0,
"end_offset": 1,
"type": "CN_CHAR",
"position": 0
},
{
"token": "是",
"start_offset": 1,
"end_offset": 2,
"type": "CN_CHAR",
"position": 1
},
{
"token": "一名",
"start_offset": 2,
"end_offset": 4,
"type": "CN_WORD",
"position": 2
},
{
"token": "一",
"start_offset": 2,
"end_offset": 3,
"type": "TYPE_CNUM",
"position": 3
},
{
"token": "名",
"start_offset": 3,
"end_offset": 4,
"type": "COUNT",
"position": 4
},
{
"token": "小",
"start_offset": 4,
"end_offset": 5,
"type": "CN_CHAR",
"position": 5
},
{
"token": "正",
"start_offset": 5,
"end_offset": 6,
"type": "CN_CHAR",
"position": 6
},
{
"token": "太",
"start_offset": 6,
"end_offset": 7,
"type": "CN_CHAR",
"position": 7
}
]
}如果我们需要小正太分词也分一个词在数据库新增
es日期打印
再次测试分词结果
{
"tokens": [
{
"token": "我",
"start_offset": 0,
"end_offset": 1,
"type": "CN_CHAR",
"position": 0
},
{
"token": "是",
"start_offset": 1,
"end_offset": 2,
"type": "CN_CHAR",
"position": 1
},
{
"token": "一名",
"start_offset": 2,
"end_offset": 4,
"type": "CN_WORD",
"position": 2
},
{
"token": "一",
"start_offset": 2,
"end_offset": 3,
"type": "TYPE_CNUM",
"position": 3
},
{
"token": "名",
"start_offset": 3,
"end_offset": 4,
"type": "COUNT",
"position": 4
},
{
"token": "小正太",
"start_offset": 4,
"end_offset": 7,
"type": "CN_WORD",
"position": 5
}
]
}可以看到小正太分成了一个词
可能遇到的问题
启动报错:Plugin [analysis-ik] was built for Elasticsearch version 6.5.0 but version 6.5.1 is running
因为要求es版本和ik版本要完全一致,可以尝试一下修改ik目录下的plugin-descriptor.properties
改成es版本
找不到数据库驱动
ikpom增加数据库驱动依赖 es lib放入数据库驱动jar
The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.
permission java.net.SocketPermission "ip:port", "listen,accept,connect,resolve";
修改jre下的lib/security java.policy
我的是在:/Library/Java/JavaVirtualMachines/jdk1.8.0_181.jdk/Contents/Home/jre/lib/security
增加:
permission java.net.SocketPermission "ip:port", "listen,accept,connect,resolve";
可能会出现当前文件只读 切换为root权限修改即可