Merge pull request #110 from JuliaLinearAlgebra:krc/channelbuffers031

Adapt to ChannelBuffers v0.3.1

Author: KlausC

Project.toml

@@ -1,6 +1,6 @@
 name = "MatrixDepot"
 uuid = "b51810bb-c9f3-55da-ae3c-350fc1fbce05"
-version = "1.0.9"
+version = "1.0.10"
 
 [deps]
 ChannelBuffers = "79a69506-cdd1-4876-b8e5-7af85e53af4f"
@@ -14,7 +14,7 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [compat]
-ChannelBuffers = "0.2, 0.3"
+ChannelBuffers = "0.3.1"
 DataFrames = "0.20, 0.22, 0.21, 1.1, 1.2, 1.3, 1.4, 1.5"
 MAT = "0.7, 0.8, 0.9, 0.10"
 Scratch = "1"

src/download.jl

@@ -133,7 +133,7 @@ function loadmatrix(data::RemoteMatrixData)
     try
         @info("downloading: $url")
         pipe = downloadpipeline(url, dirt)
-        wait(run(pipe))
+        run(pipe)
         cd(dirt)
         for file in readdir(dirt)
             mv(file, joinpath(dir, file), force=true)
@@ -164,14 +164,12 @@ function loadinfo(data::RemoteMatrixData)
         return 0
     end
     url = redirect(dataurl(data))
-    io = ChannelPipe()
-    pipe = downloadpipeline(url, io)
-    tl = run(pipe)
+    io = open(downloadpipeline(url))
     out = IOBuffer()
     s = try
         @info("downloading head of $url")
         skip = 0
-        while ( s = readskip(io) ) != ""
+        while ( s = readline(io) ) != ""
             skip = s[1] == '%'  || isempty(strip(s)) ? 0 : skip + 1
             skip <= 1 && println(out, s)
             if skip == 1 && length(split(s)) == 3
@@ -183,7 +181,7 @@ function loadinfo(data::RemoteMatrixData)
         ex isa InterruptException && rethrow()
         String(take!(out))
     finally
-        close(out)
+        close(io)
     end
     if !isempty(s)
         mkpath(dirname(file))
@@ -197,11 +195,13 @@ end
 loadinfo(data::MatrixData) = 0
 
 """
-    downloadpipeline(url)
+    downloadpipeline(url, path=nothing)
 
 Set up a command pipeline (external processes to download and expand data)
+If a path name is given, extract tar file into the empty directory or, if
+not a tar file, write contents to file of that name.
 """
-function downloadpipeline(url::AbstractString, dir=nothing)
+function downloadpipeline(url::AbstractString, dir::Union{Nothing,AbstractString}=nothing)
     urls = rsplit(url, '.', limit=3)
     cmd = Any[ChannelBuffers.curl(url)]
     if urls[end] == "gz"
@@ -216,8 +216,8 @@ function downloadpipeline(url::AbstractString, dir=nothing)
             file = rsplit(url, '/', limit=2)
             push!(cmd, joinpath(dir, file[end]))
         end
-    elseif dir isa ChannelPipe
-        push!(cmd, dir)
+    elseif dir === nothing && urls[end] == "tar"
+        push!(cmd, tarxO())
     end
     pipeline(cmd...)
 end
@@ -321,6 +321,6 @@ issvdok(::MatrixData) = false
 Copy file from remote or local url. Works around julia Downloads #69 and #36
 """
 function downloadfile(url::AbstractString, out::AbstractString)
-    wait(run(downloadcommand(url, out)))
+    run(downloadcommand(url, out))
     nothing
 end

src/matrixmarket.jl

@@ -32,7 +32,7 @@ const HERMITIAN = "hermitian"
 const SKEW_SYMMETRIC = "skew-symmetric"
 
 function mmread(file::IO)
-    line = lowercase(readskip(file))
+    line = lowercase(readline(file))
     tokens = split(line)
     if length(tokens) < 2 || tokens[1] != MATRIXM
         parserr(string("Matrixmarket: invalid header:", line))
@@ -241,7 +241,7 @@ function mmreadcomment(filename::AbstractString)
     open(filename,"r") do mmfile
         skip = 0
         while !eof(mmfile)
-            s = readskip(mmfile)
+            s = readline(mmfile)
             skip = isempty(strip(s)) || s[1] == '%' ? 0 : skip + 1
             skip <= 1 && println(io, s)
             if skip == 1
@@ -254,21 +254,6 @@ function mmreadcomment(filename::AbstractString)
     String(take!(io))
 end
 
-"""
-    readskip(io)
-
-Read a line. If line starts with a ustar-header block remove this block.
-This allows to read through a tar archive without extracting the files.
-"""
-function readskip(io::IO)
-    max = 512
-    line = readline(io)
-    if length(line) >= max && line[258:263] == "ustar\0"
-        line = line[max+1:end]
-    end
-    line
-end
-
 """
     mmreadheader(filename)
 Read header information from mtx file.
@@ -279,7 +264,7 @@ function mmreadheader(file::AbstractString)
           if stat(io).size == 0
             return nothing
           end
-          line = lowercase(readskip(io))
+          line = lowercase(readline(io))
           while true
             token = split(line)
             if length(token) >= 4 &&
@@ -291,7 +276,7 @@ function mmreadheader(file::AbstractString)
                 field = :none
                 while startswith(line, '%') || isempty(strip(line))
                     field = push_hdr!(hdr, line, field)
-                    line = readskip(io)
+                    line = readline(io)
                 end
                 res = try parseint(line) catch; [] end
                 if length(res) != (token[3] == COORD ? 3 : 2)
@@ -314,7 +299,7 @@ function mmreadheader(file::AbstractString)
                     return hdr
                 else
                     while !eof(io) && !startswith(line, '%')
-                        line = readskip(io)
+                        line = readline(io)
                     end
                     if eof(io)
                         return hdr