Add wordlist

Author: grantjenks

tests/test_coverage.py

@@ -1,7 +1,9 @@
 import os
 import sys
 from .context import wordsegment
-from wordsegment import clean, load, main, isegment, segment, UNIGRAMS, BIGRAMS
+from wordsegment import (
+    clean, load, main, isegment, segment, UNIGRAMS, BIGRAMS, WORDS,
+)
 
 load()
 
@@ -95,3 +97,8 @@ def test_main():
     main(['tests/test.txt'])
     result = os.linesep.join(('choose spain', 'this is a test')) + os.linesep
     assert sys.stdout.getvalue() == result
+
+def test_words():
+    assert len(WORDS) > 0
+    assert WORDS[0] == 'aa'
+    assert WORDS[-1] == 'zzz'

wordsegment/__init__.py

@@ -49,13 +49,18 @@ class Segmenter(object):
     )
     TOTAL = 1024908267229.0
     LIMIT = 24
+    WORDS_FILENAME = op.join(
+        op.dirname(op.realpath(__file__)),
+        'words.txt',
+    )
 
 
     def __init__(self):
         self.unigrams = {}
         self.bigrams = {}
         self.total = 0.0
         self.limit = 0
+        self.words = []
 
 
     def load(self):
@@ -64,6 +69,9 @@ def load(self):
         self.bigrams.update(self.parse(self.BIGRAMS_FILENAME))
         self.total = self.TOTAL
         self.limit = self.LIMIT
+        with io.open(self.WORDS_FILENAME, encoding='utf-8') as reader:
+            text = reader.read()
+            self.words.extend(text.splitlines())
 
 
     @staticmethod
@@ -181,6 +189,7 @@ def clean(cls, text):
 segment = _segmenter.segment    # pylint: disable=invalid-name
 UNIGRAMS = _segmenter.unigrams
 BIGRAMS = _segmenter.bigrams
+WORDS = _segmenter.words
 
 
 def main(arguments=()):