在應用程式中使用
maven坐标：
com.chenlb.mmseg4j
mmseg4j-core
1.10.0
預設加載詞典的路徑代碼如下(源碼單詞拼寫有錯誤，将就着看吧，readonly)：
另外，可以建立自己的詞庫，檔案名為words*.dic，并且檔案要以UTF-8無BOM格式編碼。
* 每個分詞檔案必須以words開頭，.dic結尾，如：words-canmou.dic
* 每個分詞檔案大小必須控制在50M以内，否則很可能會OOM
public static File getDefalutPath() {
if(defalutPath == null) {
String defPath = System.getProperty("mmseg.dic.path");
log.info("look up in mmseg.dic.path="+defPath);
if(defPath == null) {
URL url = Dictionary.class.getClassLoader().getResource("data");
if(url != null) {
defPath = url.getFile();
log.info("look up in classpath="+defPath);
} else {
defPath = System.getProperty("user.dir")+"/data";
log.info("look up in user.dir="+defPath);
}
}
defalutPath = new File(defPath);
if(!defalutPath.exists()) {
log.warning("defalut dic path="+defalutPath+" not exist");
}
}
return defalutPath;
}
建立以下工具類：
package com.caiya.software.service.utils;
import com.chenlb.mmseg4j.*;
import org.apache.commons.lang3.StringUtils;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
public class MMSeg4jUtils {
public static final String mode_simple = "simple";
public static final String mode_complex = "complex";
public static final String mode_max_word = "max-word";
public static final String mode_default = mode_max_word;
public static final String operator_default = " ";
public static Dictionary dictionary_default = Dictionary.getInstance();
public static Dictionary getDictionary(String path){
if(StringUtils.isBlank(path)){
throw new IllegalArgumentException("詞典目錄不可為空");
}
return Dictionary.getInstance(path);
}
private static Seg getSeg(String mode, Dictionary dic) {
if(mode.equals(mode_simple)){
return new SimpleSeg(dic);
}else if(mode.equals(mode_complex)){
return new ComplexSeg(dic);
}else{
return new MaxWordSeg(dic);
}
}
private static String segWords(Reader input, String wordSpilt, String mode, Dictionary dic) throws IOException {
StringBuilder sb = new StringBuilder();
Seg seg = getSeg(mode, dic);//取得不同的分詞具體算法
MMSeg mmSeg = new MMSeg(input, seg);
Word word = null;
boolean first = true;
while((word=mmSeg.next())!=null) {
if(!first) {
sb.append(wordSpilt);
}
String w = word.getString();
sb.append(w);
first = false;
}
return sb.toString();
}
public static String segWords(String txt, String wordSpilt, String mode, Dictionary dic) throws IOException {
return segWords(new StringReader(txt), wordSpilt, mode, dic);
}
public static String segWords(String txt, String wordSpilt) throws IOException {
return segWords(txt, wordSpilt, mode_default, dictionary_default);
}
public static Set segWordsSet(String txt, String wordSpilt, String mode, Dictionary dic) throws IOException {
Set sets = new HashSet();
String segWords = segWords(new StringReader(txt), wordSpilt, mode, dic);
for (String s : segWords.split(wordSpilt)){
sets.add(s);
}
return sets;
}
public static Set segWordsSet(String txt, String mode, Dictionary dic) throws IOException {
return segWordsSet(txt, operator_default, mode, dic);
}
public static Set segWordsSet(String txt) throws IOException {
return segWordsSet(txt, mode_default, dictionary_default);
}
}
調用方式：
SolrQuery params = new SolrQuery();
StringBuffer buffer = new StringBuffer();
if (StringUtils.isNotBlank(softwareQuery.getKeyWord())) {
String kw = softwareQuery.getKeyWord();
try {
// kw = MMSeg4jUtils.segWords(kw, " OR ");
kw = MMSeg4jUtils.segWords(kw, " OR ", MMSeg4jUtils.mode_complex, SoftwareConstants.dictionary);
} catch (IOException e) {
logger.error("mmseg4j切詞出現異常", e);
}
buffer.append("title:(").append(kw).append(")^50");
buffer.append(" OR tag_name:(").append(kw).append(")^1.7");
buffer.append(" OR author:(").append(kw).append(")^1.1");
buffer.append(" OR origin:(").append(kw).append(")^1.1");
buffer.append(" OR description:(").append(kw).append(")^1.1");
buffer.append(" OR txt:(").append(kw).append(")^1.1");
} else {
buffer.append("*:*");
}
params.setQuery(buffer.toString());
//其中， // 初始化詞典檔案 /Users/caiya/workspace/test/dic
// dictionary = MMSeg4jUtils.getDictionary(dicPath);
分詞效果：
擴充詞典内容：
$ cat words.dic
十衣素
韓都衣舍專賣旗艦旗艦店
十衣素啊哈哈
由此可見，一般的分詞器都是基于正向最大比對，稍後我們可以最這塊進行拓展和挖掘。
在solr中的使用
目前solr 4.7.2使用mmseg4j 2.0.0版本
dicPath 指定詞庫位置(每個MMSegTokenizerFactory可以指定不同的目錄，當是相對目錄時，是相對 solr.home 的目錄)，mode 指定分詞模式(simple|complex|max-word，預設是max-word)。