From b09e6b99f92d40998e4f78082e31967e8ac80a92 Mon Sep 17 00:00:00 2001 From: jinrui Date: Mon, 25 Nov 2024 20:08:04 +0800 Subject: [PATCH] feat: rspack_cacheable support rspack_sources::BoxSource (#8527) * feat: rspack_cacheable support rspack_sources::BoxSource * fix: ci --- Cargo.lock | 1 + crates/rspack_cacheable/src/lib.rs | 1 - crates/rspack_cacheable/src/utils/mod.rs | 3 - .../src/utils/type_wrapper.rs | 108 ------------------ .../src/with/as_preset/mod.rs | 2 +- .../src/with/as_preset/rspack_source/mod.rs | 103 ----------------- .../src/with/as_preset/rspack_sources/mod.rs | 86 ++++++++++++++ crates/rspack_cacheable_test/Cargo.toml | 1 + .../tests/with/as_preset/mod.rs | 1 + .../tests/with/as_preset/rspack_sources.rs | 22 ++++ 10 files changed, 112 insertions(+), 216 deletions(-) delete mode 100644 crates/rspack_cacheable/src/utils/mod.rs delete mode 100644 crates/rspack_cacheable/src/utils/type_wrapper.rs delete mode 100644 crates/rspack_cacheable/src/with/as_preset/rspack_source/mod.rs create mode 100644 crates/rspack_cacheable/src/with/as_preset/rspack_sources/mod.rs create mode 100644 crates/rspack_cacheable_test/tests/with/as_preset/rspack_sources.rs diff --git a/Cargo.lock b/Cargo.lock index cbe920881c2..86ba268dc0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3669,6 +3669,7 @@ dependencies = [ "once_cell", "rspack_cacheable", "rspack_resolver", + "rspack_sources", "rustc-hash 1.1.0", "serde_json", "swc_core", diff --git a/crates/rspack_cacheable/src/lib.rs b/crates/rspack_cacheable/src/lib.rs index aaf1941ed65..9d09d29b261 100644 --- a/crates/rspack_cacheable/src/lib.rs +++ b/crates/rspack_cacheable/src/lib.rs @@ -3,7 +3,6 @@ pub use rspack_macros::{cacheable, cacheable_dyn}; #[cfg(feature = "noop")] pub use rspack_macros::{disable_cacheable as cacheable, disable_cacheable_dyn as cacheable_dyn}; pub mod r#dyn; -pub mod utils; pub mod with; mod context; diff --git a/crates/rspack_cacheable/src/utils/mod.rs b/crates/rspack_cacheable/src/utils/mod.rs deleted file mode 100644 index 9e149f6f4cd..00000000000 --- a/crates/rspack_cacheable/src/utils/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -mod type_wrapper; - -pub use type_wrapper::{TypeWrapper, TypeWrapperRef}; diff --git a/crates/rspack_cacheable/src/utils/type_wrapper.rs b/crates/rspack_cacheable/src/utils/type_wrapper.rs deleted file mode 100644 index 150e9599ad5..00000000000 --- a/crates/rspack_cacheable/src/utils/type_wrapper.rs +++ /dev/null @@ -1,108 +0,0 @@ -use rkyv::{ - bytecheck::{CheckBytes, StructCheckContext}, - rancor::{Fallible, Source, Trace}, - ser::{Allocator, Writer}, - string::{ArchivedString, StringResolver}, - vec::{ArchivedVec, VecResolver}, - Archive, Deserialize, Place, Portable, Serialize, -}; - -pub struct TypeWrapperRef<'a> { - pub type_name: &'a str, - pub bytes: &'a [u8], -} - -pub struct ArchivedTypeWrapper { - type_name: ArchivedString, - bytes: ArchivedVec, -} - -unsafe impl Portable for ArchivedTypeWrapper {} - -pub struct TypeWrapper { - pub type_name: String, - pub bytes: Vec, -} - -impl<'a> Archive for TypeWrapperRef<'a> { - type Archived = ArchivedTypeWrapper; - type Resolver = (StringResolver, VecResolver); - - #[inline] - fn resolve(&self, resolver: Self::Resolver, out: Place) { - let field_ptr = unsafe { &raw mut (*out.ptr()).type_name }; - let field_out = unsafe { Place::from_field_unchecked(out, field_ptr) }; - ArchivedString::resolve_from_str(self.type_name, resolver.0, field_out); - let field_ptr = unsafe { &raw mut (*out.ptr()).bytes }; - let field_out = unsafe { Place::from_field_unchecked(out, field_ptr) }; - ArchivedVec::resolve_from_len(self.bytes.len(), resolver.1, field_out); - } -} - -impl Archive for TypeWrapper { - type Archived = ArchivedTypeWrapper; - type Resolver = (StringResolver, VecResolver); - - #[inline] - fn resolve(&self, _resolver: Self::Resolver, _out: Place) { - unreachable!() - } -} - -impl<'a, S> Serialize for TypeWrapperRef<'a> -where - S: ?Sized + Fallible + Writer + Allocator, - S::Error: Source, -{ - #[inline] - fn serialize(&self, serializer: &mut S) -> Result { - Ok(( - ArchivedString::serialize_from_str(self.type_name, serializer)?, - ArchivedVec::serialize_from_slice(self.bytes, serializer)?, - )) - } -} - -unsafe impl CheckBytes for ArchivedTypeWrapper -where - ArchivedString: CheckBytes, - ArchivedVec: CheckBytes, - C: Fallible + ?Sized, - C::Error: Trace, -{ - unsafe fn check_bytes(bytes: *const Self, context: &mut C) -> Result<(), C::Error> { - ArchivedString::check_bytes(core::ptr::addr_of!((*bytes).type_name), context).map_err(|e| { - ::trace( - e, - StructCheckContext { - struct_name: "ArchivedTypeWrapper", - field_name: "type_name", - }, - ) - })?; - ArchivedVec::::check_bytes(core::ptr::addr_of!((*bytes).bytes), context).map_err(|e| { - ::trace( - e, - StructCheckContext { - struct_name: "ArchivedTypeWrapper", - field_name: "bytes", - }, - ) - })?; - Ok(()) - } -} - -impl Deserialize for ArchivedTypeWrapper -where - D: Fallible + ?Sized, - D::Error: Source, -{ - #[inline] - fn deserialize(&self, deserializer: &mut D) -> Result { - Ok(TypeWrapper { - type_name: Deserialize::::deserialize(&self.type_name, deserializer)?, - bytes: Deserialize::, D>::deserialize(&self.bytes, deserializer)?, - }) - } -} diff --git a/crates/rspack_cacheable/src/with/as_preset/mod.rs b/crates/rspack_cacheable/src/with/as_preset/mod.rs index 6fd04954a6f..c26358bd941 100644 --- a/crates/rspack_cacheable/src/with/as_preset/mod.rs +++ b/crates/rspack_cacheable/src/with/as_preset/mod.rs @@ -2,7 +2,7 @@ mod camino; mod json; mod lightningcss; mod rspack_resolver; -mod rspack_source; +mod rspack_sources; mod serde_json; mod swc; mod ustr; diff --git a/crates/rspack_cacheable/src/with/as_preset/rspack_source/mod.rs b/crates/rspack_cacheable/src/with/as_preset/rspack_source/mod.rs deleted file mode 100644 index 89bafbe7774..00000000000 --- a/crates/rspack_cacheable/src/with/as_preset/rspack_source/mod.rs +++ /dev/null @@ -1,103 +0,0 @@ -use std::sync::Arc; - -use rkyv::{ - rancor::Fallible, - ser::{Allocator, Writer}, - vec::{ArchivedVec, VecResolver}, - with::{ArchiveWith, DeserializeWith, SerializeWith}, - Place, -}; -use rspack_sources::{ - OriginalSource, RawSource, Source, SourceMap, SourceMapSource, SourceMapSourceOptions, -}; - -use super::AsPreset; -use crate::{ - utils::{TypeWrapper, TypeWrapperRef}, - DeserializeError, SerializeError, -}; - -pub struct SourceResolver { - inner: VecResolver, - len: usize, -} - -impl ArchiveWith> for AsPreset { - type Archived = ArchivedVec; - type Resolver = SourceResolver; - - #[inline] - fn resolve_with(_field: &Arc, resolver: Self::Resolver, out: Place) { - ArchivedVec::resolve_from_len(resolver.len, resolver.inner, out) - } -} - -// TODO add cacheable to rspack-sources -impl SerializeWith, S> for AsPreset -where - S: Fallible + Writer + Allocator, -{ - fn serialize_with( - field: &Arc, - serializer: &mut S, - ) -> Result { - let inner = field.as_ref().as_any(); - let bytes = if let Some(raw_source) = inner.downcast_ref::() { - let data = TypeWrapperRef { - type_name: "RawSource", - bytes: &raw_source.buffer(), - }; - crate::to_bytes(&data, &())? - } else if let Some(original_source) = inner.downcast_ref::() { - let source = original_source.source(); - let data = Some(TypeWrapperRef { - type_name: "OriginalSource", - bytes: source.as_bytes(), - }); - crate::to_bytes(&data, &())? - } else if let Some(source_map_source) = inner.downcast_ref::() { - let source = source_map_source.source(); - let data = Some(TypeWrapperRef { - type_name: "SourceMapSource", - bytes: source.as_bytes(), - }); - crate::to_bytes(&data, &())? - } else { - return Err(SerializeError::MessageError("unsupported rspack source")); - }; - Ok(SourceResolver { - inner: ArchivedVec::serialize_from_slice(&bytes, serializer)?, - len: bytes.len(), - }) - } -} - -impl DeserializeWith, Arc, D> for AsPreset -where - D: Fallible, -{ - fn deserialize_with( - field: &ArchivedVec, - _de: &mut D, - ) -> Result, DeserializeError> { - let TypeWrapper { type_name, bytes } = crate::from_bytes(field, &())?; - match type_name.as_str() { - // TODO change to enum - "RawSource" => Ok(Arc::new(RawSource::from(bytes))), - // TODO save original source name - "OriginalSource" => Ok(Arc::new(OriginalSource::new( - "a", - String::from_utf8(bytes).expect("unexpected bytes"), - ))), - "SourceMapSource" => Ok(Arc::new(SourceMapSource::new(SourceMapSourceOptions { - value: String::from_utf8(bytes).expect("unexpected bytes"), - name: String::from("a"), - source_map: SourceMap::default(), - original_source: None, - inner_source_map: None, - remove_original_source: true, - }))), - _ => Err(DeserializeError::MessageError("unsupported box source")), - } - } -} diff --git a/crates/rspack_cacheable/src/with/as_preset/rspack_sources/mod.rs b/crates/rspack_cacheable/src/with/as_preset/rspack_sources/mod.rs new file mode 100644 index 00000000000..c91cea71193 --- /dev/null +++ b/crates/rspack_cacheable/src/with/as_preset/rspack_sources/mod.rs @@ -0,0 +1,86 @@ +use rkyv::{ + rancor::Fallible, + ser::{Allocator, Writer}, + with::{ArchiveWith, DeserializeWith, SerializeWith}, + Archive, Archived, Deserialize, Place, Resolver, Serialize, +}; +use rspack_sources::{ + BoxSource, RawSource, Source, SourceExt, SourceMap, SourceMapSource, WithoutOriginalOptions, +}; + +use super::AsPreset; +use crate::{cacheable, DeserializeError, SerializeError}; + +#[cacheable(crate=crate)] +pub struct CacheableSource { + buffer: Vec, + map: Option, +} + +pub struct InnerResolver { + source: CacheableSource, + resolver: Resolver, +} + +impl ArchiveWith for AsPreset { + type Archived = Archived; + type Resolver = InnerResolver; + + #[inline] + fn resolve_with(_field: &BoxSource, resolver: Self::Resolver, out: Place) { + let InnerResolver { source, resolver } = resolver; + source.resolve(resolver, out) + } +} + +impl SerializeWith for AsPreset +where + S: Fallible + Allocator + Writer, +{ + fn serialize_with( + field: &BoxSource, + serializer: &mut S, + ) -> Result { + let map = match field.map(&Default::default()) { + Some(map) => Some( + map + .to_json() + .map_err(|_| SerializeError::MessageError("source map to json failed"))?, + ), + None => None, + }; + let source = CacheableSource { + buffer: field.buffer().to_vec(), + map, + }; + Ok(InnerResolver { + resolver: source.serialize(serializer)?, + source, + }) + } +} + +impl DeserializeWith, BoxSource, D> for AsPreset +where + D: Fallible, +{ + fn deserialize_with( + field: &Archived, + deserializer: &mut D, + ) -> Result { + let CacheableSource { buffer, map } = field.deserialize(deserializer)?; + if let Some(map) = &map { + if let Ok(source_map) = SourceMap::from_json(map) { + return Ok( + SourceMapSource::new(WithoutOriginalOptions { + value: String::from_utf8_lossy(&buffer), + name: "persistent-cache", + source_map, + }) + .boxed(), + ); + } + } + Ok(RawSource::from(buffer).boxed()) + } +} diff --git a/crates/rspack_cacheable_test/Cargo.toml b/crates/rspack_cacheable_test/Cargo.toml index 16ea7992afa..d99cdf48a24 100644 --- a/crates/rspack_cacheable_test/Cargo.toml +++ b/crates/rspack_cacheable_test/Cargo.toml @@ -15,6 +15,7 @@ lightningcss = { workspace = true } once_cell = { workspace = true } rspack_cacheable = { path = "../rspack_cacheable" } rspack_resolver = { workspace = true } +rspack_sources = { workspace = true } rustc-hash = { workspace = true } serde_json = { workspace = true } swc_core = { workspace = true, features = ["ecma_ast"] } diff --git a/crates/rspack_cacheable_test/tests/with/as_preset/mod.rs b/crates/rspack_cacheable_test/tests/with/as_preset/mod.rs index 4fa36e17091..6b17ac2fa69 100644 --- a/crates/rspack_cacheable_test/tests/with/as_preset/mod.rs +++ b/crates/rspack_cacheable_test/tests/with/as_preset/mod.rs @@ -2,6 +2,7 @@ mod camino; mod json; mod lightningcss; mod rspack_resolver; +mod rspack_sources; mod serde_json; mod swc; mod ustr; diff --git a/crates/rspack_cacheable_test/tests/with/as_preset/rspack_sources.rs b/crates/rspack_cacheable_test/tests/with/as_preset/rspack_sources.rs new file mode 100644 index 00000000000..7f1c1a538ae --- /dev/null +++ b/crates/rspack_cacheable_test/tests/with/as_preset/rspack_sources.rs @@ -0,0 +1,22 @@ +use rspack_cacheable::{cacheable, from_bytes, to_bytes, with::AsPreset}; +use rspack_sources::{BoxSource, RawSource, SourceExt}; + +#[cacheable] +#[derive(Debug)] +struct Data(#[cacheable(with=AsPreset)] BoxSource); + +#[test] +fn test_rspack_source() { + fn test_data(data: Data) { + let bytes = to_bytes(&data, &()).unwrap(); + let new_data: Data = from_bytes(&bytes, &()).unwrap(); + assert_eq!(data.0.buffer(), new_data.0.buffer()); + assert_eq!( + data.0.map(&Default::default()), + new_data.0.map(&Default::default()) + ); + } + + test_data(Data(RawSource::from("123".as_bytes()).boxed())); + test_data(Data(RawSource::from("123").boxed())); +}