python基础之停用词过滤详解

def load_dictionary(path):
    map=JClass('java.util.TreeMap')()
    with open(path,encoding='utf-8') as src:
        for word in src:
            word=word.strip()
            map[word]=word
    return JClass('com.hankcs.hanlp.collection.trie.DoubleArrayTrie')(map)

def remove_stopwords(termlist,trie):
    return [term.word for term in termlist if not trie.containsKey(term.word)]

if __name__ == "__main__":
    HanLP.Config.ShowTermNature=False
    trie=load_dictionary(HanLP.Config.CoreStopWordDictionaryPath)
    text="今天就这样吧！明天我们在说可以吗？"
    segment=DoubleArrayTrieSegment()
    termlist=segment.seg(text)
    print("分词结果",termlist)
    print("去掉停用词",remove_stopwords(termlist,trie))

#直接过滤方法
def direct_remove_stopwords(text,replacement,trie):
    jstring=JClass('java.lang.String')
    searcher=trie.getLongestSearcher(JString(text),0)
    offset=0
    result=''
    while searcher.next():
        begin=searcher.begin
        end=begin+searcher.length
        if begin>offset:
            result+=text[offset:begin]
            result+=replacement
            offset=end
    if offset<len(text):
        result+=text[offset:]
    return result


if __name__ == "__main__":
    HanLP.Config.ShowTermNature = False
    trie = load_dictionary(HanLP.Config.CoreStopWordDictionaryPath)
    text = "今天就这样吧！明天我们在说可以吗？"
    segment = DoubleArrayTrieSegment()
    termlist = segment.seg(text)
    print("分词结果", termlist)
    print("去掉停用词", remove_stopwords(termlist, trie))
    print("不分词去掉停用词", direct_remove_stopwords(text, "**", trie))