Merge branch 'main' into vec_graph-copy

Author: progval

.github/workflows/pipeline.yml

@@ -0,0 +1,22 @@
+name: LLP Pipeline
+
+on:
+  push:
+    branches: ["main"]
+  pull_request:
+    branches: ["main"]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Adding baremetal arm target
+        run: rustup target add aarch64-unknown-none
+      - name: Copy Cnr-2000 to a new directory
+        run: mkdir llp && cp tests/data/cnr-2000.graph ./llp && cp tests/data/cnr-2000.properties ./llp
+      - name: Run test pipeline on cnr-2000
+        run: ./pipeline.sh ./llp/cnr-2000

.mailmap

@@ -0,0 +1,5 @@
+Matteo Dell'Acqua <[email protected]>
+Tommaso Fontana <[email protected]>
+Tommaso Fontana <[email protected]> <[email protected]>
+Tommaso Fontana <[email protected]> <[email protected]>
+Valentin Lorentz <[email protected]>

Cargo.toml

@@ -11,7 +11,8 @@ categories = ["compression", "data-structures"]
 
 
 [features]
-default = []
+default = ["cli"]
+cli = ["dep:clap", "dep:clap_complete", "dep:env_logger"] # Enable the compilation of the webgraph binary
 slow_tests = [] # Test feature that enables long running tests
 be_bins = [] # Enable read / write of only BE bvgraphs (to reduce code size)
 le_bins = [] # Enable read / write of only LE bvgraphs (to reduce code size)
@@ -21,24 +22,16 @@ fuzz = ["dep:arbitrary", "dep:zip", "dsi-bitstream/fuzz"] # Expose the fuzzing h
 anyhow = { version = "1.0.79", features=["backtrace"]}
 java-properties = "2.0.0"
 mmap-rs = "0.6.1"
-bitvec = { version = "1.0.1", features = ["atomic"] }
 num_cpus = "1.16.0"
 epserde = "0.4.0"
-#sux = "0.2.0"
-sux = {git = "https://github.com/vigna/sux-rs"}
-dsi-bitstream = "0.3.0"
-#dsi-bitstream = {git = "https://github.com/vigna/dsi-bitstream-rs"}
-clap = { version = "4.4.18", features = ["derive", "string"] }
-clap_complete = "4.4.10"
-dsi-progress-logger = "0.2.2"
+sux = "0.3.1"
+dsi-bitstream = "0.4.0"
+dsi-progress-logger = "0.2.4"
 log = "0.4.20"
-stderrlog = "0.5.4"
 rand = { version = "0.8.5", features = ["small_rng"] }
 rayon = "1.8.1"
 tempfile = "3.5.0"
 bytemuck = "1.14.0"
-arbitrary = { version = "1.3.2", features = ["derive"], optional = true }
-zip = {version="0.6.6", optional=true}
 libc = "0.2.147"
 itertools = "0.12.0"
 lender = "0.2.9"
@@ -49,17 +42,27 @@ dary_heap = "0.3.6"
 rdst = { version  ="0.20.12", features = ["multi-threaded"] }
 sealed = "0.5.0"
 
-[dev-dependencies]
-rand = { version = "0.8.5", features = ["small_rng"] }
-env_logger = "0.11.0"
+# Cli
+clap = { version = "4.4.18", features = ["derive", "string"], optional = true }
+clap_complete = {version = "4.4.10", optional = true}
+env_logger = {version = "0.11.0", optional = true}
+
+# Fuzzing deps
+arbitrary = { version = "1.3.2", features = ["derive"], optional = true }
+zip = {version="0.6.6", optional=true}
+predicates = "3.1.0"
+sysinfo = "0.30.8"
+
+[build-dependencies]
+built = { version = "0.7", features= ["chrono", "git2"] }
 
 [profile.release] # Used for the examples
 opt-level = 3             # like --release
-lto = "fat"               # Full LTO
+#lto = "fat"               # Full LTO
 overflow-checks = false   # Disable integer overflow checks.
 debug = true              # Include debug info.
 debug-assertions = false  # Enables debug assertions.
-codegen-units=1           # slower compile times, but maybe better perf
+#codegen-units=1           # slower compile times, but maybe better perf
 
 [lib]
 name = "webgraph"

README.md

@@ -4,53 +4,138 @@
 [![dependents](https://img.shields.io/librariesio/dependents/cargo/webgraph)](https://crates.io/crates/webgraph/reverse_dependencies)
 ![GitHub CI](https://github.com/vigna/webgraph-rs/actions/workflows/rust.yml/badge.svg)
 ![license](https://img.shields.io/crates/l/webgraph)
-[![](https://tokei.rs/b1/github/vigna/webgraph-rs)](https://github.com/vigna/webgraph-rs).
+[![](https://tokei.rs/b1/github/vigna/webgraph-rs)](https://github.com/vigna/webgraph-rs)
+[![Latest version](https://img.shields.io/crates/v/webgraph.svg)](https://crates.io/crates/webgraph)
+[![Documentation](https://docs.rs/webgraph/badge.svg)](https://docs.rs/webgraph)
 
-A Rust implementation of the [WebGraph framework](https://webgraph.di.unimi.it/)
-for graph compression.
+A Rust implementation of the [WebGraph framework] for graph compression.
 
+WebGraph is a framework for graph compression aimed at studying web graphs, but
+currently being applied to several other type of graphs. It
+provides simple ways to manage very large graphs, exploiting modern compression
+techniques. More precisely, it is currently made of:
+
+- A set of simple codes, called ζ _codes_, which are particularly suitable for
+  storing web graphs (or, in general, integers with a power-law distribution in a
+  certain exponent range).
+
+- Algorithms for compressing web graphs that exploit gap compression and
+  differential compression (à la
+  [LINK](http://www.hpl.hp.com/techreports/Compaq-DEC/SRC-RR-175.html)),
+  intervalisation, and ζ codes to provide a high compression ratio (see [our
+  datasets](http://law.di.unimi.it/datasets.php)). The algorithms are controlled
+  by several parameters, which provide different tradeoffs between access speed
+  and compression ratio.
+
+- Algorithms for accessing a compressed graph without actually decompressing
+  it, using lazy techniques that delay the decompression until it is actually
+  necessary.
+
+- Algorithms for analysing very large graphs, such as {@link
+  it.unimi.dsi.webgraph.algo.HyperBall}, which has been used to show that
+  Facebook has just [four degrees of
+  separation](http://vigna.di.unimi.it/papers.php#BBRFDS).
+
+- A [Java implementation](http://webgraph.di.unimi.it/) of the algorithms above,
+  now in maintenance mode.
+
+- This crate, providing a complete, documented implementation of the algorithms
+  above in Rust. It is free software distributed under either the  [GNU Lesser
+  General Public License
+  2.1+](https://www.gnu.org/licenses/old-licenses/lgpl-2.1.html) or the [Apache
+  Software License 2.0](https://www.apache.org/licenses/LICENSE-2.0).
+
+- [Data sets](http://law.di.unimi.it/datasets.php) for large graph (e.g.,
+  billions of links).
+
+## Citation
+
+You are welcome to use and improve WebGraph for your research work! If you find
+our software useful for research, please cite the following papers in your own:
+
+- [“WebGraph: The Next Generation (Is in
+  Rust)”](http://vigna.di.unimi.it/papers.php#FVZWNG), by Tommaso Fontana,
+  Sebastiano Vigna, and Stefano Zacchiroli, in WWW '24: Companion Proceedings
+  of the ACM on Web Conference 2024, pages 686-689.  [DOI
+  10.1145/3589335.3651581](https://dl.acm.org/doi/10.1145/3589335.3651581)
+
+- [“The WebGraph Framework I: Compression
+  Techniques”](http://vigna.di.unimi.it/papers.php#BoVWFI), by Paolo Boldi and
+  Sebastiano Vigna, in _Proc. of the 13th international conference on World
+  Wide Web, WWW 2004, pages 595-602, ACM. [DOI
+  10.1145/988672.988752](https://dl.acm.org/doi/10.1145/988672.988752)
+  
 ## Quick Setup
 
 Assuming you have built all binaries, you will first need a graph in BV format,
-for example downloading it from the [LAW website](http://law.di.unimi.it/). You
-will need the `.graph` file (the bitstream containing a compressed representation
-of the graph), the `.properties` file (metadata) and the `.offsets` file (a
-bitstream containing pointers into the graph bitstream). As a first step, if
-you need random access to the successors of a node, you need
-to build an [Elias--Fano](sux::dict::EliasFano) representation of the
-offsets with the command `build_ef` (this part can be skipped if you just need
-sequential access), which will generate an `.ef` file. Then, to load a graph
-with basename `BASENAME` you need to call
+for example downloading it from the [LAW website]. For a graph with basename
+BASENAME, you will need the `BASENAME.graph` file (the bitstream containing a
+compressed representation of the graph), the `BASENAME.properties` file
+(metadata) and the `BASENAME.offsets` file (a bitstream containing pointers into
+the graph bitstream).
+
+As a first step, if you need random access to the successors of a node, you need
+to build an [Elias–Fano] representation of the offsets (this part can be skipped
+if you just need sequential access). There is a CLI command `webgraph` with many
+subcommands, among which `build`, and `webgraph build ef BASENAME` will build
+the representation for you, serializing it with [ε-serde] in a file
+named `BASENAME.ef`.
+
+Then, to load the graph you need to call
 
 ```[ignore]
 let graph = BVGraph::with_basename("BASENAME").load()?;
 ```
 
-The [`with_basename`] method returns a [`LoadConfig`] instance that can be further
-customized, selecting endianness, type of memory access, etc. By default you
-will get big endianness, memory mapping for both the graph and the offsets, and
-dynamic code dispatch.
+The [`with_basename`] method returns a [`LoadConfig`] instance that can be
+further customized, selecting endianness, type of memory access, and so on. By
+default you will get big endianness, memory mapping for both the graph and the
+offsets, and dynamic code dispatch.
+
+Once you load the graph, you can [retrieve the successors of a node] or
+[iterate on the whole graph]. In particular, using the handy [`for_`] macro,
+you can write an iteration on the graph as
 
-Once you loaded the [graph](), you can [retrieve the successors of a node]()
-or [iterate on the whole graph]().
+```[ignore]
+for_!((src, succ) in graph {
+    for dst in succ {
+        [do something with the arc src -> dst]
+    }
+});
+```
 
 ## More Options
 
-- By starting from the [`BVGraphSeq`] class you can obtain an instance that
-does not need the `.ef` file, but provides only [iteration]().
+- By starting from the [`BVGraphSeq`] class you can obtain an instance that does
+  not need the `BASENAME.ef` file, but provides only [iteration].
 
-- Graphs can be labeled by [zipping]() then together with a [labeling](). In fact,
+- Graphs can be labeled by [zipping] them together with a [labeling]. In fact,
   graphs are just labelings with `usize` labels.
 
 ## Operating on Graphs
 
-There are many operations available on graphs, such as [`transpose`] or [`simplify`].
+There are many operations available on graphs, such as [`transpose`] and
+[`simplify`]. You can [permute] a graph.
 
 ## Acknowledgments
 
-This software has been partially supported by project SERICS (PE00000014) under the NRRP MUR program funded by the EU - NGEU,
-and by project ANR COREGRAPHIE, grant ANR-20-CE23-0002 of the French Agence Nationale de la Recherche.
+This software has been partially supported by project SERICS (PE00000014) under
+the NRRP MUR program funded by the EU - NGEU, and by project ANR COREGRAPHIE,
+grant ANR-20-CE23-0002 of the French Agence Nationale de la Recherche.
 
-[`LoadConfig`]:
-[`with_basename`]:
-[`transpose`]
+[`transpose`]: <https://docs.rs/webgraph/latest/webgraph/transform/transpose/index.html>
+[`simplify`]: <https://docs.rs/webgraph/latest/webgraph/transform/simplify/index.html>
+[`with_basename`]: <https://docs.rs/webgraph/latest/webgraph/struct.BVGraph.html#method.with_basename>
+[`BVGraphSeq`]: <https://docs.rs/webgraph/latest/webgraph/struct.BVGraphSeq.html>
+[`LoadConfig`]: <https://docs.rs/webgraph/latest/webgraph/struct.LoadConfig.html>
+[iterate on the whole graph]: <https://docs.rs/webgraph/latest/webgraph/trait/SequentialLabeling.html#method.iter>
+[zipping]: <https://docs.rs/webgraph/latest/webgraph/struct/Zip.html>
+[labeling]: <https://docs.rs/webgraph/latest/webgraph/trait/SequentialLabeling.html>
+[iteration]: <https://docs.rs/webgraph/latest/webgraph/trait/SequentialLabeling.html#method.iter>
+[retrieve the successors of a node]: <https://docs.rs/webgraph/latest/webgraph/trait/RandomAccessGraph.html#method.successors>
+[LAW website]: <http://law.di.unimi.it/>
+[Elias–Fano]: <sux::dict::EliasFano>
+[WebGraph framework]: <https://webgraph.di.unimi.it/>
+[permute]: <https://docs.rs/webgraph/latest/webgraph/transform/permute/index.html>
+[ε-serde]: <nttps://crates.io/crates/epserde/>
+[`for_`]: <https://docs.rs/lender/latest/lender/macro.for_.html>

build.rs

@@ -0,0 +1,3 @@
+fn main() {
+    built::write_built_file().expect("Failed to acquire build-time information")
+}

examples/bench_sort_pairs.rs

@@ -12,12 +12,13 @@ use clap::Parser;
 use dsi_bitstream::traits::BitRead;
 use dsi_bitstream::traits::BitWrite;
 use dsi_bitstream::traits::Endianness;
-use dsi_progress_logger::*;
+use dsi_progress_logger::prelude::{ProgressLog, ProgressLogger};
 use rand::rngs::SmallRng;
 use rand::RngCore;
 use rand::SeedableRng;
 use tempfile::Builder;
 use webgraph::prelude::*;
+
 #[derive(Parser, Debug)]
 #[command(about = "Tests the merge speed of SortPairs", long_about = None)]
 struct Args {
@@ -50,14 +51,13 @@ impl<E: Endianness, W: BitRead<E>> BitDeserializer<E, W> for Mock {
     }
 }
 
+#[allow(dead_code)] // I have no idea why this happens https://github.com/rust-lang/rust/issues/12327
 pub fn main() -> Result<()> {
     let args = Args::parse();
 
-    stderrlog::new()
-        .verbosity(2)
-        .timestamp(stderrlog::Timestamp::Second)
-        .init()
-        .unwrap();
+    env_logger::builder()
+        .filter_level(log::LevelFilter::Info)
+        .try_init()?;
 
     let dir = Builder::new().prefix("bench_sort_pairs").tempdir()?;
 
@@ -83,7 +83,6 @@ pub fn main() -> Result<()> {
             pl.light_update();
         }
         pl.done();
-        return Ok(());
     } else {
         let mut sp = SortPairs::new(args.batch, dir.path())?;
 

examples/bench_swh_labels.rs

@@ -7,11 +7,11 @@
 
 use anyhow::Result;
 use clap::Parser;
-use dsi_progress_logger::*;
+use dsi_progress_logger::prelude::*;
 use lender::*;
 use std::hint::black_box;
 use std::path::PathBuf;
-use webgraph::labels::swh_labels::SwhLabels;
+use webgraph::prelude::swh_labels::SwhLabels;
 use webgraph::prelude::*;
 
 #[derive(Parser, Debug)]
@@ -24,11 +24,9 @@ struct Args {
 pub fn main() -> Result<()> {
     let args = Args::parse();
 
-    stderrlog::new()
-        .verbosity(2)
-        .timestamp(stderrlog::Timestamp::Second)
-        .init()
-        .unwrap();
+    env_logger::builder()
+        .filter_level(log::LevelFilter::Info)
+        .try_init()?;
 
     let labels = SwhLabels::load_from_file(7, &args.basename)?;
 

examples/bench_unit_graph.rs

@@ -10,7 +10,7 @@ mod bench_sort_pairs;
 use anyhow::Result;
 use clap::Parser;
 use dsi_bitstream::prelude::*;
-use dsi_progress_logger::*;
+use dsi_progress_logger::prelude::*;
 use lender::*;
 use std::hint::black_box;
 use std::path::PathBuf;
@@ -96,10 +96,9 @@ where
 pub fn main() -> Result<()> {
     let args = Args::parse();
 
-    stderrlog::new()
-        .verbosity(2)
-        .timestamp(stderrlog::Timestamp::Second)
-        .init()?;
+    env_logger::builder()
+        .filter_level(log::LevelFilter::Info)
+        .try_init()?;
 
     match get_endianness(&args.basename)?.as_str() {
         #[cfg(any(