Skip to content

perf: disable WAL for transpiled source cache #18084

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Mar 22, 2023
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 18 additions & 23 deletions cli/cache/check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use deno_core::error::AnyError;
use deno_runtime::deno_webstorage::rusqlite::params;
use deno_runtime::deno_webstorage::rusqlite::Connection;

use super::common::run_sqlite_pragma;
use super::common::INITIAL_PRAGMAS;

/// The cache used to tell whether type checking should occur again.
///
Expand Down Expand Up @@ -60,7 +60,6 @@ impl TypeCheckCache {
conn: Connection,
cli_version: String,
) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?;
create_tables(&conn, cli_version)?;

Ok(Self(Some(conn)))
Expand Down Expand Up @@ -162,27 +161,23 @@ fn create_tables(
conn: &Connection,
cli_version: String,
) -> Result<(), AnyError> {
// INT doesn't store up to u64, so use TEXT
conn.execute(
"CREATE TABLE IF NOT EXISTS checkcache (
check_hash TEXT PRIMARY KEY
)",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS tsbuildinfo (
specifier TEXT PRIMARY KEY,
text TEXT NOT NULL
)",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
let query = format!(
"{INITIAL_PRAGMAS}
-- INT doesn't store up to u64, so use TEXT
CREATE TABLE IF NOT EXISTS checkcache (
check_hash TEXT PRIMARY KEY
);
CREATE TABLE IF NOT EXISTS tsbuildinfo (
specifier TEXT PRIMARY KEY,
text TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
",
);
conn.execute_batch(&query)?;

// delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn
Expand Down
29 changes: 10 additions & 19 deletions cli/cache/common.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,6 @@

use std::hash::Hasher;

use deno_core::error::AnyError;
use deno_runtime::deno_webstorage::rusqlite::Connection;

/// A very fast insecure hasher that uses the xxHash algorithm.
#[derive(Default)]
pub struct FastInsecureHasher(twox_hash::XxHash64);
Expand Down Expand Up @@ -47,19 +44,13 @@ impl FastInsecureHasher {
}
}

/// Runs the common sqlite pragma.
pub fn run_sqlite_pragma(conn: &Connection) -> Result<(), AnyError> {
// Enable write-ahead-logging and tweak some other stuff
let initial_pragmas = "
-- enable write-ahead-logging mode
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
PRAGMA temp_store=memory;
PRAGMA page_size=4096;
PRAGMA mmap_size=6000000;
PRAGMA optimize;
";

conn.execute_batch(initial_pragmas)?;
Ok(())
}
// Disable write-ahead-logging and tweak some other stuff
pub static INITIAL_PRAGMAS: &str = "
-- disable write-ahead-logging mode
PRAGMA journal_mode=OFF;
PRAGMA synchronous=NORMAL;
PRAGMA temp_store=memory;
PRAGMA page_size=4096;
PRAGMA mmap_size=6000000;
PRAGMA optimize;
";
33 changes: 15 additions & 18 deletions cli/cache/incremental.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ use deno_runtime::deno_webstorage::rusqlite::Connection;
use serde::Serialize;
use tokio::task::JoinHandle;

use super::common::run_sqlite_pragma;
use super::common::FastInsecureHasher;
use super::common::INITIAL_PRAGMAS;

/// Cache used to skip formatting/linting a file again when we
/// know it is already formatted or has no lint diagnostics.
Expand Down Expand Up @@ -174,7 +174,6 @@ impl SqlIncrementalCache {
state_hash: u64,
cli_version: String,
) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?;
create_tables(&conn, cli_version)?;

Ok(Self { conn, state_hash })
Expand Down Expand Up @@ -242,22 +241,20 @@ fn create_tables(
conn: &Connection,
cli_version: String,
) -> Result<(), AnyError> {
// INT doesn't store up to u64, so use TEXT
conn.execute(
"CREATE TABLE IF NOT EXISTS incrementalcache (
file_path TEXT PRIMARY KEY,
state_hash TEXT NOT NULL,
source_hash TEXT NOT NULL
)",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
let query = format!(
"{INITIAL_PRAGMAS}
-- // INT doesn't store up to u64, so use TEXT
CREATE TABLE IF NOT EXISTS incrementalcache (
file_path TEXT PRIMARY KEY,
state_hash TEXT NOT NULL,
source_hash TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);"
);
conn.execute_batch(&query)?;

// delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn
Expand Down
62 changes: 26 additions & 36 deletions cli/cache/node.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use serde::Serialize;
use std::path::PathBuf;
use std::sync::Arc;

use super::common::run_sqlite_pragma;
use super::common::INITIAL_PRAGMAS;
use super::FastInsecureHasher;

// todo(dsherret): use deno_ast::CjsAnalysisData directly when upgrading deno_ast
Expand Down Expand Up @@ -155,7 +155,6 @@ impl NodeAnalysisCacheInner {
conn: Connection,
version: String,
) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?;
create_tables(&conn, &version)?;

Ok(Self { conn })
Expand Down Expand Up @@ -261,40 +260,31 @@ impl NodeAnalysisCacheInner {
}

fn create_tables(conn: &Connection, cli_version: &str) -> Result<(), AnyError> {
// INT doesn't store up to u64, so use TEXT for source_hash
conn.execute(
"CREATE TABLE IF NOT EXISTS cjsanalysiscache (
specifier TEXT PRIMARY KEY,
source_hash TEXT NOT NULL,
data TEXT NOT NULL
)",
[],
)?;
conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS cjsanalysiscacheidx
ON cjsanalysiscache(specifier)",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS esmglobalscache (
specifier TEXT PRIMARY KEY,
source_hash TEXT NOT NULL,
data TEXT NOT NULL
)",
[],
)?;
conn.execute(
"CREATE UNIQUE INDEX IF NOT EXISTS esmglobalscacheidx
ON esmglobalscache(specifier)",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
let query = format!(
"{INITIAL_PRAGMAS}
-- // INT doesn't store up to u64, so use TEXT for source_hash
CREATE TABLE IF NOT EXISTS cjsanalysiscache (
specifier TEXT PRIMARY KEY,
source_hash TEXT NOT NULL,
data TEXT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS cjsanalysiscacheidx
ON cjsanalysiscache(specifier);
CREATE TABLE IF NOT EXISTS esmglobalscache (
specifier TEXT PRIMARY KEY,
source_hash TEXT NOT NULL,
data TEXT NOT NULL
);
CREATE UNIQUE INDEX IF NOT EXISTS esmglobalscacheidx
ON esmglobalscache(specifier);
CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"
);

conn.execute_batch(&query)?;

// delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn
Expand Down
37 changes: 18 additions & 19 deletions cli/cache/parsed_source.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ use deno_graph::ParsedSourceStore;
use deno_runtime::deno_webstorage::rusqlite::params;
use deno_runtime::deno_webstorage::rusqlite::Connection;

use super::common::run_sqlite_pragma;
use super::common::INITIAL_PRAGMAS;
use super::FastInsecureHasher;

#[derive(Clone, Default)]
Expand Down Expand Up @@ -162,7 +162,6 @@ impl ParsedSourceCacheModuleAnalyzer {
cli_version: String,
sources: ParsedSourceCacheSources,
) -> Result<Self, AnyError> {
run_sqlite_pragma(&conn)?;
create_tables(&conn, cli_version)?;

Ok(Self { conn, sources })
Expand Down Expand Up @@ -292,23 +291,23 @@ fn create_tables(
conn: &Connection,
cli_version: String,
) -> Result<(), AnyError> {
// INT doesn't store up to u64, so use TEXT for source_hash
conn.execute(
"CREATE TABLE IF NOT EXISTS moduleinfocache (
specifier TEXT PRIMARY KEY,
media_type TEXT NOT NULL,
source_hash TEXT NOT NULL,
module_info TEXT NOT NULL
)",
[],
)?;
conn.execute(
"CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
)",
[],
)?;
let query = format!(
"{INITIAL_PRAGMAS}
-- INT doesn't store up to u64, so use TEXT for source_hash
CREATE TABLE IF NOT EXISTS moduleinfocache (
specifier TEXT PRIMARY KEY,
media_type TEXT NOT NULL,
source_hash TEXT NOT NULL,
module_info TEXT NOT NULL
);
CREATE TABLE IF NOT EXISTS info (
key TEXT PRIMARY KEY,
value TEXT NOT NULL
);
"
);

conn.execute_batch(&query)?;

// delete the cache when the CLI version changes
let data_cli_version: Option<String> = conn
Expand Down