Turbopack: speed up server node entrypoint chunking (#70399)

Chunk each layout/page segment separately to deduplicate work across pages Fixes PACK-3272
vercel · Sep 24, 2024 · 572bfb4 · 572bfb4
1 parent 16ffa03
commit 572bfb4
Show file tree

Hide file tree

Showing 10 changed files with 276 additions and 204 deletions.
diff --git a/crates/next-api/src/app.rs b/crates/next-api/src/app.rs
diff --git a/crates/next-api/src/instrumentation.rs b/crates/next-api/src/instrumentation.rs
@@ -155,6 +155,7 @@ impl InstrumentationEndpoint {
                     this.project.next_mode(),
                 )
                 .resolve_entries(this.asset_context),
+                OutputAssets::empty(),
                 Value::new(AvailabilityInfo::Root),
             )
             .await?;

diff --git a/crates/next-api/src/pages.rs b/crates/next-api/src/pages.rs
@@ -861,6 +861,7 @@ impl PageEndpoint {
                         ssr_entry_chunk_path,
                         ssr_module,
                         runtime_entries,
+                        OutputAssets::empty(),
                         Value::new(AvailabilityInfo::Root),
                     )
                     .await?;

diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs
@@ -478,6 +478,7 @@ impl ChunkingContext for BrowserChunkingContext {
         _path: Vc<FileSystemPath>,
         _module: Vc<Box<dyn Module>>,
         _evaluatable_assets: Vc<EvaluatableAssets>,
+        _extra_chunks: Vc<OutputAssets>,
         _availability_info: Value<AvailabilityInfo>,
     ) -> Result<Vc<EntryChunkGroupResult>> {
         bail!("Browser chunking context does not support entry chunk groups")

diff --git a/turbopack/crates/turbopack-cli/src/build/mod.rs b/turbopack/crates/turbopack-cli/src/build/mod.rs
@@ -21,7 +21,7 @@ use turbopack_core::{
     environment::{BrowserEnvironment, Environment, ExecutionEnvironment},
     issue::{handle_issues, IssueReporter, IssueSeverity},
     module::Module,
-    output::OutputAsset,
+    output::{OutputAsset, OutputAssets},
     reference::all_assets_from_entries,
     reference_type::{EntryReferenceSubType, ReferenceType},
     resolve::{
@@ -283,6 +283,7 @@ async fn build_internal(
                                     .with_extension("entry.js".into()),
                                 Vc::upcast(ecmascript),
                                 EvaluatableAssets::one(Vc::upcast(ecmascript)),
+                                OutputAssets::empty(),
                                 Value::new(AvailabilityInfo::Root),
                             )
                             .await?

diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs
@@ -104,13 +104,15 @@ pub trait ChunkingContext {
     ) -> Vc<ChunkGroupResult>;
 
     /// Generates an output chunk that:
+    /// * loads the given extra_chunks in addition to the generated chunks; and
     /// * evaluates the given assets; and
     /// * exports the result of evaluating the given module as a CommonJS default export.
     fn entry_chunk_group(
         self: Vc<Self>,
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
         evaluatable_assets: Vc<EvaluatableAssets>,
+        extra_chunks: Vc<OutputAssets>,
         availability_info: Value<AvailabilityInfo>,
     ) -> Result<Vc<EntryChunkGroupResult>>;
 
@@ -153,6 +155,7 @@ pub trait ChunkingContextExt {
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
         evaluatable_assets: Vc<EvaluatableAssets>,
+        extra_chunks: Vc<OutputAssets>,
         availability_info: Value<AvailabilityInfo>,
     ) -> Vc<Box<dyn OutputAsset>>
     where
@@ -162,6 +165,7 @@ pub trait ChunkingContextExt {
         self: Vc<Self>,
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
+        extra_chunks: Vc<OutputAssets>,
         evaluatable_assets: Vc<EvaluatableAssets>,
     ) -> Vc<EntryChunkGroupResult>
     where
@@ -171,6 +175,7 @@ pub trait ChunkingContextExt {
         self: Vc<Self>,
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
+        extra_chunks: Vc<OutputAssets>,
         evaluatable_assets: Vc<EvaluatableAssets>,
     ) -> Vc<Box<dyn OutputAsset>>
     where
@@ -219,13 +224,15 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
         evaluatable_assets: Vc<EvaluatableAssets>,
+        extra_chunks: Vc<OutputAssets>,
         availability_info: Value<AvailabilityInfo>,
     ) -> Vc<Box<dyn OutputAsset>> {
         entry_chunk_group_asset(
             path,
             Vc::upcast(self),
             module,
             evaluatable_assets,
+            extra_chunks,
             availability_info,
         )
     }
@@ -234,12 +241,14 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
         self: Vc<Self>,
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
+        extra_chunks: Vc<OutputAssets>,
         evaluatable_assets: Vc<EvaluatableAssets>,
     ) -> Vc<EntryChunkGroupResult> {
         self.entry_chunk_group(
             path,
             module,
             evaluatable_assets,
+            extra_chunks,
             Value::new(AvailabilityInfo::Root),
         )
     }
@@ -248,13 +257,15 @@ impl<T: ChunkingContext + Send + Upcast<Box<dyn ChunkingContext>>> ChunkingConte
         self: Vc<Self>,
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
+        extra_chunks: Vc<OutputAssets>,
         evaluatable_assets: Vc<EvaluatableAssets>,
     ) -> Vc<Box<dyn OutputAsset>> {
         entry_chunk_group_asset(
             path,
             Vc::upcast(self),
             module,
             evaluatable_assets,
+            extra_chunks,
             Value::new(AvailabilityInfo::Root),
         )
     }
@@ -295,10 +306,17 @@ async fn entry_chunk_group_asset(
     chunking_context: Vc<Box<dyn ChunkingContext>>,
     module: Vc<Box<dyn Module>>,
     evaluatable_assets: Vc<EvaluatableAssets>,
+    extra_chunks: Vc<OutputAssets>,
     availability_info: Value<AvailabilityInfo>,
 ) -> Result<Vc<Box<dyn OutputAsset>>> {
     Ok(chunking_context
-        .entry_chunk_group(path, module, evaluatable_assets, availability_info)
+        .entry_chunk_group(
+            path,
+            module,
+            evaluatable_assets,
+            extra_chunks,
+            availability_info,
+        )
         .await?
         .asset)
 }

diff --git a/turbopack/crates/turbopack-node/src/evaluate.rs b/turbopack/crates/turbopack-node/src/evaluate.rs
@@ -29,6 +29,7 @@ use turbopack_core::{
     ident::AssetIdent,
     issue::{Issue, IssueExt, IssueStage, OptionStyledString, StyledString},
     module::Module,
+    output::OutputAssets,
     reference_type::{InnerAssets, ReferenceType},
     virtual_source::VirtualSource,
 };
@@ -150,8 +151,12 @@ pub async fn get_evaluate_pool(
         Vc::<EvaluatableAssets>::cell(entries)
     };
 
-    let bootstrap =
-        chunking_context.root_entry_chunk_group_asset(path, entry_module, runtime_entries);
+    let bootstrap = chunking_context.root_entry_chunk_group_asset(
+        path,
+        entry_module,
+        OutputAssets::empty(),
+        runtime_entries,
+    );
 
     let output_root: Vc<FileSystemPath> = chunking_context.output_root();
     let _ = emit_package_json(output_root);

diff --git a/turbopack/crates/turbopack-node/src/lib.rs b/turbopack/crates/turbopack-node/src/lib.rs
@@ -18,7 +18,7 @@ use turbopack_core::{
     asset::{Asset, AssetContent},
     chunk::{ChunkingContext, ChunkingContextExt, EvaluatableAssets},
     module::Module,
-    output::{OutputAsset, OutputAssetsSet},
+    output::{OutputAsset, OutputAssets, OutputAssetsSet},
     source_map::GenerateSourceMap,
     virtual_output::VirtualOutputAsset,
 };
@@ -265,6 +265,7 @@ pub async fn get_intermediate_asset(
     Ok(Vc::upcast(chunking_context.root_entry_chunk_group_asset(
         chunking_context.chunk_path(main_entry.ident(), ".js".into()),
         main_entry,
+        OutputAssets::empty(),
         other_entries,
     )))
 }

diff --git a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs
@@ -15,7 +15,7 @@ use turbopack_core::{
     environment::Environment,
     ident::AssetIdent,
     module::Module,
-    output::OutputAsset,
+    output::{OutputAsset, OutputAssets},
 };
 use turbopack_ecmascript::{
     async_chunk::module::AsyncLoaderModule,
@@ -304,6 +304,7 @@ impl ChunkingContext for NodeJsChunkingContext {
         path: Vc<FileSystemPath>,
         module: Vc<Box<dyn Module>>,
         evaluatable_assets: Vc<EvaluatableAssets>,
+        extra_chunks: Vc<OutputAssets>,
         availability_info: Value<AvailabilityInfo>,
     ) -> Result<Vc<EntryChunkGroupResult>> {
         let availability_info = availability_info.into_value();
@@ -323,9 +324,11 @@ impl ChunkingContext for NodeJsChunkingContext {
         )
         .await?;
 
-        let other_chunks: Vec<_> = chunks
+        let extra_chunks = extra_chunks.await?;
+        let other_chunks: Vec<_> = extra_chunks
             .iter()
-            .map(|chunk| self.generate_chunk(*chunk))
+            .copied()
+            .chain(chunks.iter().map(|chunk| self.generate_chunk(*chunk)))
             .collect();
 
         let Some(module) = Vc::try_resolve_downcast(module).await? else {

diff --git a/turbopack/crates/turbopack-tests/tests/snapshot.rs b/turbopack/crates/turbopack-tests/tests/snapshot.rs
@@ -45,7 +45,7 @@ use turbopack_core::{
     free_var_references,
     issue::{Issue, IssueDescriptionExt},
     module::Module,
-    output::OutputAsset,
+    output::{OutputAsset, OutputAssets},
     reference_type::{EntryReferenceSubType, ReferenceType},
     source::Source,
 };
@@ -397,6 +397,7 @@ async fn run_test(resource: RcStr) -> Result<Vc<FileSystemPath>> {
                             runtime_entries
                                 .unwrap_or_else(EvaluatableAssets::empty)
                                 .with_entry(Vc::upcast(ecmascript)),
+                            OutputAssets::empty(),
                             Value::new(AvailabilityInfo::Root),
                         )
                         .await?