今天小编就为大家分享一篇python正向最大匹配分词和逆向最大匹配分词的实例，具有很好的参考价值，希望对大家有所帮助。一起跟随小编过来看看吧
正向最大匹配
# -*- coding:utf-8 -*- CODEC='utf-8' def u(s, encoding): 'converted other encoding to unicode encoding' if isinstance(s, unicode): return s else: return unicode(s, encoding) def fwd_mm_seg(wordDict, maxLen, str): 'forward max match segment' wordList = [] segStr = str segStrLen = len(segStr) for word in wordDict: print 'word: ', word print "\n" while segStrLen > 0: if segStrLen > maxLen: wordLen = maxLen else: wordLen = segStrLen subStr = segStr[0:wordLen] print "subStr: ", subStr while wordLen > 1: if subStr in wordDict: print "subStr1: %r" % subStr break else: print "subStr2: %r" % subStr wordLen = wordLen - 1 subStr = subStr[0:wordLen] # print "subStr3: ", subStr wordList.append(subStr) segStr = segStr[wordLen:] segStrLen = segStrLen - wordLen for wordstr in wordList: print "wordstr: ", wordstr return wordList def main(): fp_dict = open('words.dic') wordDict = {} for eachWord in fp_dict: wordDict[u(eachWord.strip(), 'utf-8')] = 1 segStr = u'你好世界hello world' print segStr wordList = fwd_mm_seg(wordDict, 10, segStr) print "==".join(wordList) if __name__ == '__main__': main()
逆向最大匹配
# -*- coding:utf-8 -*- def u(s, encoding): 'converted other encoding to unicode encoding' if isinstance(s, unicode): return s else: return unicode(s, encoding) CODEC='utf-8' def bwd_mm_seg(wordDict, maxLen, str): 'forward max match segment' wordList = [] segStr = str segStrLen = len(segStr) for word in wordDict: print 'word: ', word print "\n" while segStrLen > 0: if segStrLen > maxLen: wordLen = maxLen else: wordLen = segStrLen subStr = segStr[-wordLen:None] print "subStr: ", subStr while wordLen > 1: if subStr in wordDict: print "subStr1: %r" % subSt<em style="color:transparent">来源[email protected]搞@^&代*@码)网</em>r break else: print "subStr2: %r" % subStr wordLen = wordLen - 1 subStr = subStr[-wordLen:None] # print "subStr3: ", subStr wordList.append(subStr) segStr = segStr[0: -wordLen] segStrLen = segStrLen - wordLen wordList.reverse() for wordstr in wordList: print "wordstr: ", wordstr return wordList def main(): fp_dict = open('words.dic') wordDict = {} for eachWord in fp_dict: wordDict[u(eachWord.strip(), 'utf-8')] = 1 segStr = ur'你好世界hello world' print segStr wordList = bwd_mm_seg(wordDict, 10, segStr) print "==".join(wordList) if __name__ == '__main__': main()
以上这篇python正向最大匹配分词和逆向最大匹配分词的实例就是小编分享给大家的全部内容了，希望能给大家一个参考，也希望大家多多支持gaodaima搞代码网。
以上就是python正向最大匹配分词和逆向最大匹配分词的实例的详细内容，更多请关注gaodaima搞代码网其它相关文章！
搞代码网（gaodaima.com）提供的所有资源部分来自互联网，如果有侵犯您的版权或其他权益，请说明详细缘由并提供版权或权益证明然后发送到邮箱[email protected]，我们会在看到邮件的第一时间内为您处理，或直接联系QQ：872152909。本网站采用BY-NC-SA协议进行授权
转载请注明原文链接：python正向最大匹配分词和逆向最大匹配分词的实例
转载请注明原文链接：python正向最大匹配分词和逆向最大匹配分词的实例
