feat(dev-overlay): Add a query parameter to restart endpoint to invalidate the persistent cache

Author: bgw

crates/napi/src/next_api/project.rs

@@ -567,6 +567,18 @@ pub async fn project_update(
     Ok(())
 }
 
+/// Invalidates the persistent cache so that it will be deleted next time that a turbopack project
+/// is created with persistent caching enabled.
+#[napi]
+pub async fn project_invalidate_persistent_cache(
+    #[napi(ts_arg_type = "{ __napiType: \"Project\" }")] project: External<ProjectInstance>,
+) -> napi::Result<()> {
+    tokio::task::spawn_blocking(move || project.turbo_tasks.invalidate_persistent_cache())
+        .await
+        .context("panicked while invalidating persistent cache")??;
+    Ok(())
+}
+
 /// Runs exit handlers for the project registered using the [`ExitHandler`] API.
 ///
 /// This is called by `project_shutdown`, so if you're calling that API, you shouldn't call this

crates/napi/src/next_api/utils.rs

@@ -149,6 +149,14 @@ impl NextTurboTasks {
             }
         }
     }
+
+    pub fn invalidate_persistent_cache(&self) -> Result<()> {
+        match self {
+            NextTurboTasks::Memory(_) => {}
+            NextTurboTasks::PersistentCaching(turbo_tasks) => turbo_tasks.backend().invalidate()?,
+        }
+        Ok(())
+    }
 }
 
 pub fn create_turbo_tasks(

packages/next/src/build/swc/generated-native.d.ts

@@ -201,6 +201,13 @@ export declare function projectUpdate(
   project: { __napiType: 'Project' },
   options: NapiPartialProjectOptions
 ): Promise<void>
+/**
+ * Invalidates the persistent cache so that it will be deleted next time that a turbopack project
+ * is created with persistent caching enabled.
+ */
+export declare function projectInvalidatePersistentCache(project: {
+  __napiType: 'Project'
+}): Promise<void>
 /**
  * Runs exit handlers for the project registered using the [`ExitHandler`] API.
  *

packages/next/src/build/swc/index.ts

@@ -722,6 +722,10 @@ function bindingToApi(
       )
     }
 
+    invalidatePersistentCache(): Promise<void> {
+      return binding.projectInvalidatePersistentCache(this._nativeProject)
+    }
+
     shutdown(): Promise<void> {
       return binding.projectShutdown(this._nativeProject)
     }

packages/next/src/build/swc/types.ts

@@ -227,6 +227,8 @@ export interface Project {
     TurbopackResult<CompilationEvent>
   >
 
+  invalidatePersistentCache(): Promise<void>
+
   shutdown(): Promise<void>
 
   onExit(): Promise<void>

packages/next/src/client/components/react-dev-overlay/server/restart-dev-server-middleware.ts

@@ -2,20 +2,33 @@ import type { ServerResponse, IncomingMessage } from 'http'
 import type { Telemetry } from '../../../../telemetry/storage'
 import { RESTART_EXIT_CODE } from '../../../../server/lib/utils'
 import { middlewareResponse } from './middleware-response'
+import type { Project } from '../../../../build/swc/types'
 
 const EVENT_DEV_OVERLAY_RESTART_SERVER = 'DEV_OVERLAY_RESTART_SERVER'
 
-export function getRestartDevServerMiddleware(telemetry: Telemetry) {
+interface RestartDevServerMiddlewareConfig {
+  telemetry: Telemetry
+  turbopackProject?: Project
+}
+
+export function getRestartDevServerMiddleware({
+  telemetry,
+  turbopackProject,
+}: RestartDevServerMiddlewareConfig) {
   return async function (
     req: IncomingMessage,
     res: ServerResponse,
     next: () => void
   ): Promise<void> {
-    const { pathname } = new URL(`http://n${req.url}`)
+    const { pathname, searchParams } = new URL(`http://n${req.url}`)
     if (pathname !== '/__nextjs_restart_dev' || req.method !== 'POST') {
       return next()
     }
 
+    if (searchParams.has('invalidatePersistentCache')) {
+      await turbopackProject?.invalidatePersistentCache()
+    }
+
     telemetry.record({
       eventName: EVENT_DEV_OVERLAY_RESTART_SERVER,
       payload: {},

packages/next/src/server/dev/hot-reloader-turbopack.ts

@@ -649,7 +649,10 @@ export async function createHotReloaderTurbopack(
     getNextErrorFeedbackMiddleware(opts.telemetry),
     getDevOverlayFontMiddleware(),
     getDisableDevIndicatorMiddleware(),
-    getRestartDevServerMiddleware(opts.telemetry),
+    getRestartDevServerMiddleware({
+      telemetry: opts.telemetry,
+      turbopackProject: project,
+    }),
   ]
 
   const versionInfoPromise = getVersionInfo()

packages/next/src/server/dev/hot-reloader-webpack.ts

@@ -1570,7 +1570,9 @@ export default class HotReloaderWebpack implements NextJsHotReloaderInterface {
       getNextErrorFeedbackMiddleware(this.telemetry),
       getDevOverlayFontMiddleware(),
       getDisableDevIndicatorMiddleware(),
-      getRestartDevServerMiddleware(this.telemetry),
+      getRestartDevServerMiddleware({
+        telemetry: this.telemetry,
+      }),
     ]
   }
 

turbopack/crates/turbo-persistence/src/db.rs

@@ -176,7 +176,7 @@ impl TurboPersistence {
         Ok(db)
     }
 
-    /// Performas the initial check on the database directory.
+    /// Performs the initial check on the database directory.
     fn open_directory(&mut self) -> Result<()> {
         match fs::read_dir(&self.path) {
             Ok(entries) => {

turbopack/crates/turbo-tasks-backend/src/backend/mod.rs

@@ -202,6 +202,10 @@ impl<B: BackingStorage> TurboTasksBackend<B> {
             backing_storage,
         )))
     }
+
+    pub fn invalidate(&self) -> Result<()> {
+        self.0.backing_storage.invalidate()
+    }
 }
 
 impl<B: BackingStorage> TurboTasksBackendInner<B> {

turbopack/crates/turbo-tasks-backend/src/backing_storage.rs

@@ -63,6 +63,16 @@ pub trait BackingStorage: 'static + Send + Sync {
         category: TaskDataCategory,
     ) -> Vec<CachedDataItem>;
 
+    /// Called when the database should be invalidated upon re-initialization.
+    ///
+    /// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh
+    /// database. If this happens, there's no point in writing anything else to disk, or flushing
+    /// during [`KeyValueDatabase::shutdown`].
+    ///
+    /// This can be implemented by calling [`crate::database::db_invalidation::invalidate_db`] with
+    /// the database's non-versioned base path.
+    fn invalidate(&self) -> Result<()>;
+
     fn shutdown(&self) -> Result<()> {
         Ok(())
     }

turbopack/crates/turbo-tasks-backend/src/database/db_invalidation.rs

@@ -0,0 +1,54 @@
+use std::{
+    fs::{self, read_dir},
+    io,
+    path::Path,
+};
+
+const INVALIDATION_MARKER: &str = "__turbo_tasks_invalidated_db";
+
+/// Atomically write an invalidation marker.
+///
+/// Because attempting to delete currently open database files could cause issues, actual deletion
+/// of files is deferred until the next start-up (in [`check_db_invalidation`]).
+///
+/// In the case that no database is currently open (e.g. via a separate CLI subcommand), you should
+/// call [`cleanup_db`] after this to eagerly remove the database files.
+///
+/// This should be run with the base (non-versioned) path, as that likely aligns closest with user
+/// expectations (e.g. if they're clearing the cache for disk space reasons).
+pub fn invalidate_db(base_path: &Path) -> io::Result<()> {
+    fs::write(base_path.join(INVALIDATION_MARKER), &[0u8; 0])?;
+    Ok(())
+}
+
+/// Called during startup. See if the db is in a partially-completed invalidation state. Find and
+/// delete any invalidated database files.
+///
+/// This should be run with the base (non-versioned) path.
+pub fn check_db_invalidation_and_cleanup(base_path: &Path) -> io::Result<()> {
+    if fs::exists(base_path.join(INVALIDATION_MARKER))? {
+        // if this cleanup fails, we might try to open an invalid database later, so it's best to
+        // just propagate the error here.
+        cleanup_db(base_path)?;
+    };
+    Ok(())
+}
+
+/// Helper for `invalidate_db` and `check_db_invalidation`.
+pub fn cleanup_db(base_path: &Path) -> io::Result<()> {
+    let Ok(contents) = read_dir(base_path) else {
+        return Ok(());
+    };
+
+    // delete everything except the invalidation marker
+    for entry in contents {
+        let entry = entry?;
+        if entry.file_name() != INVALIDATION_MARKER {
+            let _ = fs::remove_dir_all(entry.path());
+        }
+    }
+
+    // delete the invalidation marker last, once we're sure everything is cleaned up
+    fs::remove_file(base_path.join(INVALIDATION_MARKER))?;
+    Ok(())
+}

turbopack/crates/turbo-tasks-backend/src/database/key_value_database.rs

@@ -52,6 +52,19 @@ pub trait KeyValueDatabase {
         &self,
     ) -> Result<WriteBatch<'_, Self::SerialWriteBatch<'_>, Self::ConcurrentWriteBatch<'_>>>;
 
+    /// Called when the database has been invalidated via
+    /// [`crate::backing_storage::BackingStorage::invalidate`]
+    ///
+    /// This typically means that we'll restart the process or `turbo-tasks` soon with a fresh
+    /// database. If this happens, there's no point in writing anything else to disk, or flushing
+    /// during [`KeyValueDatabase::shutdown`].
+    ///
+    /// This is a best-effort optimization hint, and the database may choose to ignore this and
+    /// continue file writes.
+    fn prevent_writes(&self) {
+        // this is an optional performance hint to the database
+    }
+
     fn shutdown(&self) -> Result<()> {
         Ok(())
     }

turbopack/crates/turbo-tasks-backend/src/database/mod.rs

@@ -1,5 +1,6 @@
 #[cfg(feature = "lmdb")]
 mod by_key_space;
+pub mod db_invalidation;
 pub mod db_versioning;
 #[cfg(feature = "lmdb")]
 pub mod fresh_db_optimization;

turbopack/crates/turbo-tasks-backend/src/database/turbo.rs

@@ -23,8 +23,8 @@ pub struct TurboKeyValueDatabase {
 }
 
 impl TurboKeyValueDatabase {
-    pub fn new(path: PathBuf) -> Result<Self> {
-        let db = Arc::new(TurboPersistence::open(path.to_path_buf())?);
+    pub fn new(versioned_path: PathBuf) -> Result<Self> {
+        let db = Arc::new(TurboPersistence::open(versioned_path.to_path_buf())?);
         let mut this = Self {
             db: db.clone(),
             compact_join_handle: Mutex::new(None),
@@ -99,6 +99,8 @@ impl KeyValueDatabase for TurboKeyValueDatabase {
         }))
     }
 
+    fn prevent_writes(&self) {}
+
     fn shutdown(&self) -> Result<()> {
         // Wait for the compaction to finish
         if let Some(join_handle) = self.compact_join_handle.lock().take() {

turbopack/crates/turbo-tasks-backend/src/kv_backing_storage.rs

@@ -1,4 +1,4 @@
-use std::{borrow::Borrow, cmp::max, sync::Arc};
+use std::{borrow::Borrow, cmp::max, path::PathBuf, sync::Arc};
 
 use anyhow::{Context, Result, anyhow};
 use rayon::iter::{IndexedParallelIterator, IntoParallelIterator, ParallelIterator};
@@ -12,6 +12,7 @@ use crate::{
     backing_storage::BackingStorage,
     data::CachedDataItem,
     database::{
+        db_invalidation::invalidate_db,
         key_value_database::{KeySpace, KeyValueDatabase},
         write_batch::{
             BaseWriteBatch, ConcurrentWriteBatch, SerialWriteBatch, WriteBatch, WriteBatchRef,
@@ -83,11 +84,17 @@ fn as_u32(bytes: impl Borrow<[u8]>) -> Result<u32> {
 
 pub struct KeyValueDatabaseBackingStorage<T: KeyValueDatabase> {
     database: T,
+    /// Used when calling [`BackingStorage::invalidate`]. Can be `None` in the memory-only/no-op
+    /// storage case.
+    base_path: Option<PathBuf>,
 }
 
 impl<T: KeyValueDatabase> KeyValueDatabaseBackingStorage<T> {
-    pub fn new(database: T) -> Self {
-        Self { database }
+    pub fn new(database: T, base_path: Option<PathBuf>) -> Self {
+        Self {
+            database,
+            base_path,
+        }
     }
 
     fn with_tx<R>(
@@ -445,6 +452,15 @@ impl<T: KeyValueDatabase + Send + Sync + 'static> BackingStorage
             .unwrap_or_default()
     }
 
+    fn invalidate(&self) -> Result<()> {
+        if let Some(base_path) = &self.base_path {
+            // `base_path` can be `None` for a `NoopKvDb`
+            invalidate_db(base_path)?;
+            self.database.prevent_writes()
+        }
+        Ok(())
+    }
+
     fn shutdown(&self) -> Result<()> {
         self.database.shutdown()
     }

turbopack/crates/turbo-tasks-backend/src/lib.rs

@@ -14,6 +14,7 @@ mod utils;
 use std::path::Path;
 
 use anyhow::Result;
+use database::db_invalidation::check_db_invalidation_and_cleanup;
 
 use crate::database::{
     db_versioning::handle_db_versioning, noop_kv::NoopKvDb, turbo::TurboKeyValueDatabase,
@@ -37,7 +38,7 @@ pub type LmdbBackingStorage = KeyValueDatabaseBackingStorage<
 
 #[cfg(feature = "lmdb")]
 pub fn lmdb_backing_storage(
-    path: &Path,
+    base_path: &Path,
     version_info: &GitVersionInfo,
 ) -> Result<LmdbBackingStorage> {
     use crate::database::{
@@ -46,30 +47,39 @@ pub fn lmdb_backing_storage(
         startup_cache::StartupCacheLayer,
     };
 
-    let path = handle_db_versioning(path, version_info)?;
-    let fresh_db = is_fresh(&path);
-    let database = crate::database::lmdb::LmbdKeyValueDatabase::new(&path)?;
+    check_db_invalidation_and_cleanup(base_path)?;
+    let versioned_path = handle_db_versioning(base_path, version_info)?;
+    let fresh_db = is_fresh(&versioned_path);
+    let database = crate::database::lmdb::LmbdKeyValueDatabase::new(&versioned_path)?;
     let database = FreshDbOptimization::new(database, fresh_db);
-    let database = StartupCacheLayer::new(database, path.join("startup.cache"), fresh_db)?;
+    let database =
+        StartupCacheLayer::new(database, versioned_path.join("startup.cache"), fresh_db)?;
     let database = ReadTransactionCache::new(database);
-    Ok(KeyValueDatabaseBackingStorage::new(database))
+    Ok(KeyValueDatabaseBackingStorage::new(
+        database,
+        Some(base_path.to_owned()),
+    ))
 }
 
 pub type TurboBackingStorage = KeyValueDatabaseBackingStorage<TurboKeyValueDatabase>;
 
 pub fn turbo_backing_storage(
-    path: &Path,
+    base_path: &Path,
     version_info: &GitVersionInfo,
 ) -> Result<TurboBackingStorage> {
-    let path = handle_db_versioning(path, version_info)?;
-    let database = TurboKeyValueDatabase::new(path)?;
-    Ok(KeyValueDatabaseBackingStorage::new(database))
+    check_db_invalidation_and_cleanup(base_path)?;
+    let versioned_path = handle_db_versioning(base_path, version_info)?;
+    let database = TurboKeyValueDatabase::new(versioned_path)?;
+    Ok(KeyValueDatabaseBackingStorage::new(
+        database,
+        Some(base_path.to_owned()),
+    ))
 }
 
 pub type NoopBackingStorage = KeyValueDatabaseBackingStorage<NoopKvDb>;
 
 pub fn noop_backing_storage() -> NoopBackingStorage {
-    KeyValueDatabaseBackingStorage::new(NoopKvDb)
+    KeyValueDatabaseBackingStorage::new(NoopKvDb, None)
 }
 
 #[cfg(feature = "lmdb")]