Bugfixes

Author: zgornel

Manifest.toml

@@ -9,10 +9,10 @@ version = "0.2.0"
 uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
 
 [[BinaryProvider]]
-deps = ["Libdl", "Pkg", "SHA", "Test"]
-git-tree-sha1 = "055eb2690182ebc31087859c3dd8598371d3ef9e"
+deps = ["Libdl", "SHA"]
+git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648"
 uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
-version = "0.5.3"
+version = "0.5.4"
 
 [[DataStructures]]
 deps = ["InteractiveUtils", "OrderedCollections", "Random", "Serialization", "Test"]
@@ -48,9 +48,6 @@ git-tree-sha1 = "fc6ee05e35074a66dc12a716065a25d9deece6fb"
 uuid = "8ef0a80b-9436-5d2c-a485-80b904378c43"
 version = "0.4.2"
 
-[[LibGit2]]
-uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
-
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
@@ -70,22 +67,14 @@ uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
 [[OrderedCollections]]
 deps = ["Random", "Serialization", "Test"]
-git-tree-sha1 = "85619a3f3e17bb4761fe1b1fd47f0e979f964d5b"
+git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
-version = "1.0.2"
-
-[[Pkg]]
-deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
-uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.1.0"
 
 [[Printf]]
 deps = ["Unicode"]
 uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 
-[[REPL]]
-deps = ["InteractiveUtils", "Markdown", "Sockets"]
-uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
-
 [[Random]]
 deps = ["Serialization"]
 uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
@@ -117,10 +106,6 @@ version = "0.3.0"
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
-[[UUIDs]]
-deps = ["Random", "SHA"]
-uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
-
 [[Unicode]]
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 

NEWS.md

@@ -1,5 +1,9 @@
 ## StringAnalysis Release Notes
 
+v0.3.6
+------
+ - Bugfix release
+
 v0.3.5
 ------
  - Improved LSA embedding performance

Project.toml

@@ -1,7 +1,7 @@
 name = "StringAnalysis"
 uuid = "b66b7d2f-f536-51df-9f97-4dfb9d27c005"
 authors = ["Corneliu Cofaru <[email protected]>"]
-version = "0.3.5"
+version = "0.3.6"
 
 [deps]
 AutoHashEquals = "15f4f7f2-30c1-5605-9d31-71845cf9641f"
@@ -18,3 +18,6 @@ TSVD = "9449cd9e-2762-5aa3-a617-5413e99d722e"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Unicode = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 WordTokenizers = "796a5d58-b03d-544a-977e-18100b691f6e"
+
+[compat]
+julia = "1"

docs/src/examples.md

@@ -113,7 +113,11 @@ using Languages
 it = StringDocument("Quest'e un piccolo esempio di come si puo fare l'analisi");
 StringAnalysis.language!(it, Languages.Italian());
 prepare!(it, strip_articles|strip_prepositions|strip_whitespace);
-it.text
+text(it)
+```
+In the case of `AbstractString`s, the language has to be explicitly defined:
+```@repl index
+prepare("Nous sommes tous d'accord avec les examples!", stem_words, language=Languages.French())
 ```
 
 ## Features

src/preprocessing.jl

@@ -354,16 +354,16 @@ function prepare!(entity,  # can be an AbstractDocument or Corpus
         ((flags & strip_single_chars) > 0) && push!(rpatterns, strip2regex[strip_single_chars])
     end
     # known words
-    lang = _language(entity)
+    language = _language(entity)
     if (flags & strip_articles) > 0
-        union!(skip_words, articles(lang))
+        union!(skip_words, articles(language))
     else
-        ((flags & strip_indefinite_articles) > 0) && union!(skip_words, indefinite_articles(lang))
-        ((flags & strip_definite_articles) > 0) && union!(skip_words, definite_articles(lang))
+        ((flags & strip_indefinite_articles) > 0) && union!(skip_words, indefinite_articles(language))
+        ((flags & strip_definite_articles) > 0) && union!(skip_words, definite_articles(language))
     end
-    ((flags & strip_prepositions) > 0) && union!(skip_words, prepositions(lang))
-    ((flags & strip_pronouns) > 0) && union!(skip_words, pronouns(lang))
-    ((flags & strip_stopwords) > 0) && union!(skip_words, stopwords(lang))
+    ((flags & strip_prepositions) > 0) && union!(skip_words, prepositions(language))
+    ((flags & strip_pronouns) > 0) && union!(skip_words, pronouns(language))
+    ((flags & strip_stopwords) > 0) && union!(skip_words, stopwords(language))
     if !isempty(skip_words)
         push!(rpatterns, _build_words_pattern(skip_words))
     end
@@ -386,7 +386,7 @@ end
 
 function prepare(s::AbstractString,
                  flags::UInt32;
-                 lang::Language = DEFAULT_LANGUAGE,
+                 language::Language = DEFAULT_LANGUAGE,
                  skip_patterns = Vector{Regex}(),
                  skip_words = Vector{AbstractString}())
     os = s  # Initialize output string
@@ -407,14 +407,14 @@ function prepare(s::AbstractString,
     end
     # known words
     if (flags & strip_articles) > 0
-        union!(skip_words, articles(lang))
+        union!(skip_words, articles(language))
     else
-        ((flags & strip_indefinite_articles) > 0) && union!(skip_words, indefinite_articles(lang))
-        ((flags & strip_definite_articles) > 0) && union!(skip_words, definite_articles(lang))
+        ((flags & strip_indefinite_articles) > 0) && union!(skip_words, indefinite_articles(language))
+        ((flags & strip_definite_articles) > 0) && union!(skip_words, definite_articles(language))
     end
-    ((flags & strip_prepositions) > 0) && union!(skip_words, prepositions(lang))
-    ((flags & strip_pronouns) > 0) && union!(skip_words, pronouns(lang))
-    ((flags & strip_stopwords) > 0) && union!(skip_words, stopwords(lang))
+    ((flags & strip_prepositions) > 0) && union!(skip_words, prepositions(language))
+    ((flags & strip_pronouns) > 0) && union!(skip_words, pronouns(language))
+    ((flags & strip_stopwords) > 0) && union!(skip_words, stopwords(language))
     if !isempty(skip_words)
         push!(rpatterns, _build_words_pattern(skip_words))
     end
@@ -431,6 +431,6 @@ function prepare(s::AbstractString,
         os = remove_patterns(os, r)
     end
     # Stemming
-    ((flags & stem_words) > 0) && (os = stem(os))
+    ((flags & stem_words) > 0) && (os = stem(os, language=language))
     return os
 end