Merge pull request #156 from JuliaAI/dev

For a 2.2.0 release

Author: ablaom

Project.toml

@@ -1,26 +1,30 @@
 name = "ScientificTypes"
 uuid = "321657f4-b219-11e9-178b-2701a2544e81"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "2.1.3"
+version = "2.2"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
 ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
+CorpusLoaders = "214a0ac2-f95b-54f7-a80b-442ed9c2c9e8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 PersistenceDiagramsBase = "b1ad91c1-539c-4ace-90bd-ea06abc420fa"
 PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 ScientificTypesBase = "30f210dd-8aff-4c5f-94ba-8e64358c1161"
 StatisticalTraits = "64bff920-2084-43da-a3e6-9bb72801c0c9"
 Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 
 [compat]
 CategoricalArrays = "0.8, 0.9, 0.10"
 ColorTypes = "0.9, 0.10, 0.11"
+CorpusLoaders = "0.3.2"
 Distributions = "0.25"
 PersistenceDiagramsBase = "0.1"
 PrettyTables = "1"
-ScientificTypesBase = "2.1"
+Reexport = "1.2"
+ScientificTypesBase = "2.2"
 StatisticalTraits = "2"
 Tables = "1"
 julia = "1"

docs/src/index.md

@@ -51,11 +51,17 @@ ScientificTimeType
 Sampleable{Ω}
 └─ Density{Ω}
 
+Annotated{S}
+
+AnnotationFor{S}
+
+Multiset{S}
+
 Table{K}
 
 Textual
 
-PersistenceDiagram
+ManifoldPoint{MT}
 
 Unknown
 ```
@@ -109,26 +115,32 @@ typically much slower than calling `scitype` or `elscitype`.
 The table below summarizes the default convention for representing
 scientific types:
 
-Type `T`        | `scitype(x)` for `x::T`           | package required
+Type `T`        | `scitype(x)` for `x::T`           | package/module required
 :-------------- | :-------------------------------- | :------------------------
 `Missing`       | `Missing`                         |
 `Nothing`       | `Nothing`                         |
 `AbstractFloat` | `Continuous`                      |
 `Integer`       |  `Count`                          |
 `String`        | `Textual`                         |
-`CategoricalValue` | `Multiclass{N}` where `N = nlevels(x)`, provided `x.pool.ordered == false`  | CategoricalArrays
-`CategoricalString` | `Multiclass{N}` where `N = nlevels(x)`, provided `x.pool.ordered == false`  | CategoricalArrays
-`CategoricalValue` | `OrderedFactor{N}` where `N = nlevels(x)`, provided `x.pool.ordered == true`| CategoricalArrays
-`CategoricalString` | `OrderedFactor{N}` where `N = nlevels(x)` provided `x.pool.ordered == true` | CategoricalArrays
+`CategoricalValue` | `Multiclass{N}` where `N = nlevels(x)`, provided `x.pool.ordered == false`  | CategoricalArrays.jl
+`CategoricalString` | `Multiclass{N}` where `N = nlevels(x)`, provided `x.pool.ordered == false`  | CategoricalArrays.jl
+`CategoricalValue` | `OrderedFactor{N}` where `N = nlevels(x)`, provided `x.pool.ordered == true`| CategoricalArrays.jl
+`CategoricalString` | `OrderedFactor{N}` where `N = nlevels(x)` provided `x.pool.ordered == true` | CategoricalArrays.jl
 `Date`          | `ScientificDate`     | Dates
 `Time`          | `ScientificTime`     | Dates
 `DateTime`      | `ScientificDateTime` | Dates
 `Distributions.Sampleable{F,S}` | `Sampleable{Ω}` where `Ω` is scitype of sample space, according to `{F,S}` 
 `Distributions.Distributions{F,S}` | `Density{Ω}` where `Ω` is scitype of sample space, according to `{F,S}` 
-`AbstractArray{<:Gray,2}` | `GrayImage{W,H}` where `(W, H) = size(x)`                                   | ColorTypes
-`AbstractArrray{<:AbstractRGB,2}` | `ColorImage{W,H}` where `(W, H) = size(x)`                                  | ColorTypes
+`AbstractArray{<:Gray,2}` | `GrayImage{W,H}` where `(W, H) = size(x)`                                   | ColorTypes.jl
+`AbstractArrray{<:AbstractRGB,2}` | `ColorImage{W,H}` where `(W, H) = size(x)`                                  | ColorTypes.jl
 `PersistenceDiagram` | `PersistenceDiagram` | PersistenceDiagramsBase
-any table type `T` supported by Tables.jl | `Table{K}` where `K=Union{column_scitypes...}`                      | Tables
+any table type `T` supported by Tables.jl | `Table{K}` where `K=Union{column_scitypes...}`                      | Tables.jl
+† `CorpusLoaders.TaggedWord` | `Annotated{Textual}` | CorpusLoaders.jl
+† `CorpusLoaders.Document{AbstractVector{Q}}` | `Annotated{AbstractVector{Scitype(Q)}}` | CorpusLoaders.jl
+† `AbstractDict{<:AbstractString,<:Integer}` | `Multiset{Textual}` | 
+† `AbstractDict{<:TaggedWord,<:Integer}` | `Multiset{Annotated{Textual}}` | CorpusLoaders.jl
+
+† *Experimental* and subject to change in new minor or patch release
 
 Here `nlevels(x) = length(levels(x.pool))`.
 

src/ScientificTypes.jl

@@ -1,26 +1,19 @@
 module ScientificTypes
 
 # Dependencies
-using ScientificTypesBase
+using Reexport
+@reexport using ScientificTypesBase
 using Tables
 using CategoricalArrays
 using ColorTypes
 using PersistenceDiagramsBase
+using CorpusLoaders
 using PrettyTables
 using Dates
 import Distributions
 
 import StatisticalTraits: info
 
-# re-exports from ScientificTypes
-export Scientific, Found, Unknown, Known, Finite, Infinite,
-    OrderedFactor, Multiclass, Count, Continuous, Textual,
-    Binary, ColorImage, GrayImage, Image, Table,
-    ScientificTimeType, ScientificDate, ScientificDateTime,
-    ScientificTime,
-    Density, Sampleable
-export scitype, scitype_union, elscitype, nonmissing, trait
-
 # re-export from StatisticalTraits
 export info
 

src/convention/scitype.jl

@@ -66,9 +66,21 @@ ST.scitype(::Distributions.Sampleable{F,S}) where {F,S} =
 ST.scitype(::Distributions.Distribution{F,S}) where {F,S} =
     Density{space_scitype(F,S)}
 
-
-
-
+# Text analysis - EXPERIMENTAL
+
+type2scitype(T::Type) = ST.Scitype(T, DefaultConvention())
+type2scitype(::Type{<:AbstractVector{T}}) where T =
+    AbstractVector{type2scitype(T)}
+ST.scitype(::TaggedWord, ::DefaultConvention) = Annotated{Textual}
+ST.scitype(::Document{<:AbstractVector{T}}, ::DefaultConvention) where T =
+    Annotated{AbstractVector{type2scitype(T)}}
+ST.scitype(::AbstractDict{<:AbstractString,<:Integer},
+           ::DefaultConvention) = Multiset{Textual}
+ST.scitype(::AbstractDict{<:TaggedWord,<:Integer},
+           ::DefaultConvention) = Multiset{Annotated{Textual}}
+ST.scitype(::AbstractDict{<:Union{TaggedWord,AbstractString},<:Integer},
+           ::DefaultConvention) =
+               Multiset{Annotated{Textual}}
 
 # Scitype for fast array broadcasting
 
@@ -80,3 +92,5 @@ ST.Scitype(::Type{<:Date},               ::DefaultConvention) = ScientificDate
 ST.Scitype(::Type{<:Time},               ::DefaultConvention) = ScientificTime
 ST.Scitype(::Type{<:DateTime},           ::DefaultConvention) = ScientificDateTime
 ST.Scitype(::Type{<:PersistenceDiagram}, ::DefaultConvention) = PersistenceDiagram
+ST.Scitype(::Type{<:TaggedWord},         ::DefaultConvention) =
+    Annotated{Textual}

test/runtests.jl

@@ -1,6 +1,6 @@
 using Test, ScientificTypes, ScientificTypesBase, Random
 using Tables, CategoricalArrays, DataFrames
-using ColorTypes, PersistenceDiagramsBase
+using ColorTypes, PersistenceDiagramsBase, CorpusLoaders
 using Dates
 # using CSV # dropped until julia release new LTS as issue for 1.0
 import Distributions

test/scitypes.jl

@@ -240,6 +240,22 @@ end
     @test scitype(FooSampleable()) == Sampleable{Count}
 end
 
+@testset "text analysis" begin
+    tagged_word = CorpusLoaders.PosTaggedWord("NN", "wheelbarrow")
+    tagged_word2 = CorpusLoaders.PosTaggedWord("NN", "soil")
+    @test scitype(tagged_word) == Annotated{Textual}
+    bag_of_words = Dict("cat"=>1, "dog"=>3)
+    @test scitype(bag_of_words) == Multiset{Textual}
+    bag_of_tagged_words = Dict(tagged_word => 5)
+    @test scitype(bag_of_tagged_words) == Multiset{Annotated{Textual}}
+    @test scitype(Document("kadsfkj", "My Document")) == Unknown
+    @test scitype(Document([tagged_word, tagged_word2], "My Other Doc")) ==
+        Annotated{AbstractVector{Annotated{Textual}}}
+    nested_tokens = [["dog", "cat"], ["bird", "cat"]]
+    @test scitype(Document(nested_tokens), "Essay Number 1") ==
+                  Annotated{AbstractVector{AbstractVector{Textual}}}
+end
+
 @testset "Autotype+tight" begin
     x = [1,2,3,missing];
     x = x[1:3]