Use Julia packages instead of unix commands (#107)

* Use CURL_jll Tar_jll Gzip_jll instead of commands

* julia1.8 required by CURL_jll

* not Tar_jll for windows

* allow CURL_jll  8

* play with dependencies

* use ChannelBuffers for crul/gunzip/tar

* cd out of dir to be removed

* rename `metadata` to `getmeta`

* using DataFrames restricted

Author: KlausC

.github/workflows/CI.yml

@@ -15,11 +15,13 @@ jobs:
       fail-fast: false
       matrix:
         version:
-          - '1.0'
+          - '1.6'
           - '1'
           - 'nightly'
         os:
           - ubuntu-latest
+          - macos-latest
+          - windows-latest
         arch:
           - x64
     steps:

.gitignore

@@ -13,7 +13,9 @@ myMatrixDepot*
 
 *.mem
 *.cov
+lcov.info
 
+/docs/build
 # ignore generated files from Sphinx
 _build
 _static

Project.toml

@@ -1,9 +1,9 @@
 name = "MatrixDepot"
 uuid = "b51810bb-c9f3-55da-ae3c-350fc1fbce05"
-version = "1.0.8"
+version = "1.0.9"
 
 [deps]
-CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"
+ChannelBuffers = "79a69506-cdd1-4876-b8e5-7af85e53af4f"
 DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MAT = "23992714-dd62-5051-b70f-ba57cb901cac"
@@ -14,8 +14,8 @@ Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [compat]
-CodecZlib = "0.6, 0.7"
-DataFrames = "0.20, 0.22, 0.21, 1.1, 1.2, 1.3"
+ChannelBuffers = "0.2, 0.3"
+DataFrames = "0.20, 0.22, 0.21, 1.1, 1.2, 1.3, 1.4, 1.5"
 MAT = "0.7, 0.8, 0.9, 0.10"
 Scratch = "1"
 julia = "1"

src/MatrixDepot.jl

@@ -36,10 +36,10 @@ Access is like
     using MatrixDepot
 
     A = matrixdepot("hilb", 10) # locally generated hilbert matrix dimensions (10,10)
-    
+
     A = matrixdepot("HB/1138_bus")     # named matrix of the SuiteSparse Collection
     A = matrixdepot(sp(1))             # same matrix using numerical id
-    A = matrixdepot("Harwell*/*/1138_bus") # matrix from the Matrix Market Collection 
+    A = matrixdepot("Harwell*/*/1138_bus") # matrix from the Matrix Market Collection
 
     md = mdopen("*/bfly")   # named matrix with some extra data
     A = md.A
@@ -69,7 +69,6 @@ Access is like
 """
 module MatrixDepot
 using LinearAlgebra, SparseArrays, Serialization
-using CodecZlib
 using Scratch
 import Base: show
 

src/common.jl

@@ -327,7 +327,7 @@ end
 function list!(db::MatrixDatabase, res::Vector{String}, p::Not)
     isempty(res) && return res
     cres = list!(db, copy(res), p.pattern)
-    isempty(cres) && return res 
+    isempty(cres) && return res
     resall!(db, res)
     setdiff!(res, cres)
 end
@@ -348,7 +348,7 @@ end
 
 list!(db::MatrixDatabase, res::Vector{String}, ::Tuple{}) = res
 
-# logical AND 
+# logical AND
 function list!(db::MatrixDatabase, res::Vector{String}, r::Tuple)
     isempty(res) && return res
     check_symbols(r)
@@ -419,7 +419,7 @@ mdatav(db::MatrixDatabase, p::Pattern) = verify_loaded(db, mdata(db, p))
 
 Load data from remote repository for all problems matching pattern.
 
-Return the number of successfully loaded matrices. 
+Return the number of successfully loaded matrices.
 """
 load(p::Pattern) = load(MATRIX_DB, p)
 load(db::MatrixDatabase, p::Pattern) = _load(db, loadmatrix, p)
@@ -486,13 +486,13 @@ function mdata(db::MatrixDatabase, p::Pattern)
 end
 
 """
-    metadata([db, ], Union{MatrixDescriptor,MatrixData})
+    getmeta([db, ], Union{MatrixDescriptor,MatrixData})
 
 Return copy of list of metadata names.
 """
-metadata(mdesc::MatrixDescriptor) = metadata(mdesc.data)
-metadata(data::RemoteMatrixData) = copy(data.metadata)
-metadata(data::MatrixData) = String[]
+getmeta(mdesc::MatrixDescriptor) = getmeta(mdesc.data)
+getmeta(data::RemoteMatrixData) = copy(data.metadata)
+getmeta(data::MatrixData) = String[]
 
 _mdopen(data::RemoteMatrixData)= MatrixDescriptor(data)
 function _mdopen(data::GeneratedMatrixData, args...)

src/datareader.jl

@@ -99,7 +99,7 @@ Example:
 """
 metasymbols(md::MatrixDescriptor{<:RemoteMatrixData}) = metasymbols(md.data)
 function metasymbols(data::RemoteMatrixData)
-    Symbol.(metastring.(data.name, metadata(data)))
+    Symbol.(metastring.(data.name, getmeta(data)))
 end
 function metasymbols(md::MatrixDescriptor{<:GeneratedMatrixData})
     mdc = md.cache[]
@@ -118,7 +118,7 @@ end
 #internal helper to select special metadata (matrix, rhs, or solution)
 function metaname(data::RemoteMatrixData, exli::AbstractString...)
     base = rsplit(data.name, '/', limit=2)[end]
-    meda = metadata(data)
+    meda = getmeta(data)
     for ext in exli
         f = metastring_reverse(data, ext)
         if f in meda
@@ -165,12 +165,11 @@ function metastring_reverse(data::RemoteMatrixData, metaabbr::AbstractString)
     MTX = ".mtx"
     base = split(data.name, '/')[end]
     metaabbr == "A" && return string(base, MTX)
-    mdata = metadata(data)
+    mdata = getmeta(data)
     meta = string(base, '_', metaabbr)
     meta in mdata && return meta
     meta = string(meta, MTX)
     meta in mdata && return meta
     metaabbr
 end
 metastring_reverse(::MatrixData, metaabbr::AbstractString) = metaabbr
-

src/download.jl

@@ -3,6 +3,7 @@
 #####################################################
 
 using Base.Filesystem
+using ChannelBuffers
 
 # collect the keys from local database (MATRIXDICT or USERMATRIXDICT)
 # provide a numerical id counting from 1 for either database.
@@ -106,28 +107,6 @@ function update(db::MatrixDatabase=MATRIX_DB)
     downloadindices(db, ignoredb=true)
 end
 
-function gunzip(fname)
-    endswith(fname, ".gz") || error("gunzip: $fname: unknown suffix")
-
-    destname = rsplit(fname, ".gz", limit=2)[1]
-    BUFFSIZE = 1000000
-    try
-        open(destname, "w") do f
-            open(GzipDecompressorStream, fname) do g
-                buffer = read(g, BUFFSIZE)
-                while length(buffer) > 0
-                    write(f, buffer)
-                    buffer = read(g, BUFFSIZE)
-                end
-            end
-        end
-    catch
-        @warn "decompression error - file $destname set to empty."
-        open(destname, "w") do f; end
-    end
-    destname
-end
-
 """
     loadmatrix(data::RemoteMatrixData)
 
@@ -144,28 +123,26 @@ function loadmatrix(data::RemoteMatrixData)
         addmetadata!(data)
         return 0
     end
-    dirfn = localfile(data)
     dir = dirname(localdir(data))
+    dirt = string(dir, ".tmp")
+    rm(dirt, force=true, recursive=true)
+    mkpath(dirt)
     url = redirect(dataurl(data))
-    tarfile = ""
-
     isdir(dir) || mkpath(dir)
     wdir = pwd()
     try
         @info("downloading: $url")
-        downloadfile(url, dirfn)
-        tarfile = gunzip(dirfn)
-        cd(dir)
-        rfile = relpath(string(tarfile))
-        if endswith(tarfile, ".tar")
-            run(`tar -xf $rfile`)
-            rm(tarfile; force=true)
+        pipe = downloadpipeline(url, dirt)
+        wait(run(pipe))
+        cd(dirt)
+        for file in readdir(dirt)
+            mv(file, joinpath(dir, file), force=true)
         end
-    catch
-        
+    catch ex
+        @warn("download of $url failed: $ex")
     finally
         cd(wdir)
-        rm(dirfn, force=true)
+        rm(dirt, force=true, recursive=true)
     end
     addmetadata!(data)
     1
@@ -187,20 +164,19 @@ function loadinfo(data::RemoteMatrixData)
         return 0
     end
     url = redirect(dataurl(data))
-    pipe = downloadpipeline(url)
+    io = ChannelPipe()
+    pipe = downloadpipeline(url, io)
+    tl = run(pipe)
     out = IOBuffer()
     s = try
         @info("downloading head of $url")
-        open(pipe, "r") do io
-            skip = 0
-            while ( s = readline(io) ) != ""
-                skip = s[1] == '%'  || isempty(strip(s)) ? 0 : skip + 1
-                skip <= 1 && println(out, s)
-                if skip == 1 && length(split(s)) == 3
-                    break
-                end
-            end;
-            close(io)
+        skip = 0
+        while ( s = readskip(io) ) != ""
+            skip = s[1] == '%'  || isempty(strip(s)) ? 0 : skip + 1
+            skip <= 1 && println(out, s)
+            if skip == 1 && length(split(s)) == 3
+                break
+            end
         end
         String(take!(out))
     catch ex
@@ -225,22 +201,29 @@ loadinfo(data::MatrixData) = 0
 
 Set up a command pipeline (external processes to download and expand data)
 """
-function downloadpipeline(url::AbstractString)
+function downloadpipeline(url::AbstractString, dir=nothing)
     urls = rsplit(url, '.', limit=3)
-    cmd = []
-    push!(cmd, downloadcommand(url))
+    cmd = Any[ChannelBuffers.curl(url)]
     if urls[end] == "gz"
-        push!(cmd, `gzip -dc`)
+        push!(cmd, gunzip())
         resize!(urls, length(urls)-1)
+        url = url[1:end-3]
     end
-    if urls[end] == "tar"
-        push!(cmd, `tar -xOf -`)
+    if dir isa AbstractString
+        if urls[end] == "tar"
+            push!(cmd, tarx(dir))
+        else
+            file = rsplit(url, '/', limit=2)
+            push!(cmd, joinpath(dir, file[end]))
+        end
+    elseif dir isa ChannelPipe
+        push!(cmd, dir)
     end
     pipeline(cmd...)
 end
 
 function downloadcommand(url::AbstractString, filename::AbstractString="-")
-    `sh -c 'curl "'$url'" -Lso "'$filename'"'`
+    curl(url) → (filename == "-" ? stdout : filename)
 end
 
 function data_warn(data::RemoteMatrixData, dn, i1, i2)
@@ -338,7 +321,6 @@ issvdok(::MatrixData) = false
 Copy file from remote or local url. Works around julia Downloads #69 and #36
 """
 function downloadfile(url::AbstractString, out::AbstractString)
-    run(downloadcommand(url, out))
+    wait(run(downloadcommand(url, out)))
     nothing
 end
-

src/downloadsp.jl

@@ -3,7 +3,7 @@
 #
 
 using MAT
-using DataFrames
+using DataFrames: DataFrames, DataFrame, select!
 using Base.Filesystem
 
 #const SS_SITE = "https://sparse.tamu.edu"

src/matrixmarket.jl

@@ -32,9 +32,9 @@ const HERMITIAN = "hermitian"
 const SKEW_SYMMETRIC = "skew-symmetric"
 
 function mmread(file::IO)
-    line = lowercase(readline(file))
+    line = lowercase(readskip(file))
     tokens = split(line)
-    if tokens[1] != MATRIXM
+    if length(tokens) < 2 || tokens[1] != MATRIXM
         parserr(string("Matrixmarket: invalid header:", line))
     end
     line = readline(file)
@@ -241,7 +241,7 @@ function mmreadcomment(filename::AbstractString)
     open(filename,"r") do mmfile
         skip = 0
         while !eof(mmfile)
-            s = readline(mmfile)
+            s = readskip(mmfile)
             skip = isempty(strip(s)) || s[1] == '%' ? 0 : skip + 1
             skip <= 1 && println(io, s)
             if skip == 1
@@ -254,6 +254,21 @@ function mmreadcomment(filename::AbstractString)
     String(take!(io))
 end
 
+"""
+    readskip(io)
+
+Read a line. If line starts with a ustar-header block remove this block.
+This allows to read through a tar archive without extracting the files.
+"""
+function readskip(io::IO)
+    max = 512
+    line = readline(io)
+    if length(line) >= max && line[258:263] == "ustar\0"
+        line = line[max+1:end]
+    end
+    line
+end
+
 """
     mmreadheader(filename)
 Read header information from mtx file.
@@ -264,7 +279,7 @@ function mmreadheader(file::AbstractString)
           if stat(io).size == 0
             return nothing
           end
-          line = lowercase(readline(io))
+          line = lowercase(readskip(io))
           while true
             token = split(line)
             if length(token) >= 4 &&
@@ -275,8 +290,8 @@ function mmreadheader(file::AbstractString)
                 hdr = Dict{Symbol,Any}()
                 field = :none
                 while startswith(line, '%') || isempty(strip(line))
-                    field = push_hdr!(hdr, line, field) 
-                    line = readline(io)
+                    field = push_hdr!(hdr, line, field)
+                    line = readskip(io)
                 end
                 res = try parseint(line) catch; [] end
                 if length(res) != (token[3] == COORD ? 3 : 2)
@@ -299,7 +314,7 @@ function mmreadheader(file::AbstractString)
                     return hdr
                 else
                     while !eof(io) && !startswith(line, '%')
-                        line = readline(io)
+                        line = readskip(io)
                     end
                     if eof(io)
                         return hdr
@@ -456,4 +471,3 @@ function parsenext(T::Type{<:Complex}, c, p)
     p, s = parsenext(R, c, p)
     p, r + s*im
 end
-

src/types.jl

@@ -211,7 +211,7 @@ function Base.show(io::IO, data::RemoteMatrixData)
     nnz = hd.nnz == hd.dnz ? "$(hd.nnz)" : "$(hd.nnz)/$(hd.dnz)"
     print(io, " $(hd.m)x$(hd.n)($nnz) ")
     print(io, data.date != 0 ? data.date : "")
-    meta = join(metastring.(data.name, metadata(data)), ", ")
+    meta = join(metastring.(data.name, getmeta(data)), ", ")
     n = length(meta)
     if n > 40
         meta = string(meta[1:17], " ... ", meta[end-17:end])