From 6a954704654aa16ea3f758edbffafe387f50975c Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 18 Jun 2021 15:09:38 +0200 Subject: [PATCH 01/59] Initial commit --- LICENSE | 201 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 2 + 2 files changed, 203 insertions(+) create mode 100644 LICENSE create mode 100644 README.md diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..24b7969 --- /dev/null +++ b/README.md @@ -0,0 +1,2 @@ +# scales +SCALE (De)Serialization From a66619fcfa8f40069d24133a50072a759016b474 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 18 Jun 2021 15:10:27 +0200 Subject: [PATCH 02/59] Cargo init --- .gitignore | 2 ++ Cargo.toml | 9 +++++++++ src/lib.rs | 7 +++++++ 3 files changed, 18 insertions(+) create mode 100644 .gitignore create mode 100644 Cargo.toml create mode 100644 src/lib.rs diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..96ef6c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..240a783 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "scales" +version = "0.1.0" +authors = ["Daniel Olano "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..31e1bb2 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(test)] +mod tests { + #[test] + fn it_works() { + assert_eq!(2 + 2, 4); + } +} From 83a24126ffbcc1dc5a9c1c4515084d433b073f44 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 18 Jun 2021 17:16:26 +0200 Subject: [PATCH 03/59] Test simple struct serialization --- Cargo.toml | 10 ++++++++-- src/lib.rs | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 240a783..7887ce5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,12 @@ version = "0.1.0" authors = ["Daniel Olano "] edition = "2018" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +scale-info = { version = "0.6.0", default-features = false } +serde = { version = "1.0.126", default-features = false } + +[dev-dependencies] +parity-scale-codec = { version = "2.1.3", features = ["derive"] } +scale-info = { version = "0.6.0", features = ["derive"] } +serde = { version = "1.0.126", features = ["derive"] } +serde_json = "1.0.64" diff --git a/src/lib.rs b/src/lib.rs index 31e1bb2..9fbed04 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,51 @@ +use scale_info::Type; +use serde::Serialize; + +pub struct Value<'a> { + data: &'a [u8], + info: &'a Type, +} + +impl<'a> Value<'a> { + pub fn new(data: &'a [u8], info: &'a Type) -> Self { + Value { data, info } + } +} + +impl<'a> Serialize for Value<'a> { + fn serialize(&self, _serializer: S) -> Result + where + S: serde::Serializer, + { + todo!() + } +} + #[cfg(test)] mod tests { + use std::error::Error; + + use super::*; + use parity_scale_codec::Encode; + use scale_info::TypeInfo; + use serde_json::to_value; + #[test] - fn it_works() { - assert_eq!(2 + 2, 4); + fn serialize_simple_struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: u32, + baz: bool, + } + let foo = Foo { + bar: 123, + baz: true, + }; + let data = foo.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, &info); + + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) } } From ca1841f487abbad7fb1f0ba1be83416ffe5360de Mon Sep 17 00:00:00 2001 From: Stanly Johnson Date: Thu, 29 Jul 2021 17:54:22 +0530 Subject: [PATCH 04/59] first pass --- .github/workflows/test.yml | 64 ++++++++++++++++++++++++++++++++++++++ src/lib.rs | 34 +++++++++++++++++--- 2 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..51e280c --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,64 @@ +on: [push, pull_request] + +name: Continuous integration + +jobs: + check: + name: Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: check + + test: + name: Test Suite + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add rustfmt + - uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add clippy + - uses: actions-rs/cargo@v1 + with: + command: clippy + args: -- -D warnings \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 9fbed04..c3f1ac6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,5 +1,8 @@ -use scale_info::Type; -use serde::Serialize; +use std::any::Any; + +use scale_info::{Type, TypeDef}; +use serde::ser::{Error, SerializeStruct}; +use serde::{Serialize, Serializer}; pub struct Value<'a> { data: &'a [u8], @@ -13,11 +16,24 @@ impl<'a> Value<'a> { } impl<'a> Serialize for Value<'a> { - fn serialize(&self, _serializer: S) -> Result + fn serialize(&self, ser: S) -> Result where S: serde::Serializer, { - todo!() + match self.info.type_def() { + TypeDef::Primitive(u32) => ser.serialize_u32(self.data[0].into()), + TypeDef::Composite(x) => { + let fields = x.fields(); + let mut state = ser.serialize_struct("", fields.len())?; + for (i, f) in fields.iter().enumerate() { + println!("{:?}", f); + let name = f.name().unwrap(); + state.serialize_field(name, &self.data[i])?; + } + state.end() + } + _ => ser.serialize_bytes(self.data), + } } } @@ -30,6 +46,16 @@ mod tests { use scale_info::TypeInfo; use serde_json::to_value; + #[test] + fn serialize_u32() -> Result<(), Box> { + let foo = 2u32; + let data = foo.encode(); + let info = u32::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + #[test] fn serialize_simple_struct() -> Result<(), Box> { #[derive(Encode, Serialize, TypeInfo)] From 06f4c94c0688cc662e705fb9b638c4ef627acd8a Mon Sep 17 00:00:00 2001 From: Stanly Johnson Date: Tue, 3 Aug 2021 20:16:26 +0530 Subject: [PATCH 05/59] Add more types to serialize --- src/lib.rs | 211 ++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 199 insertions(+), 12 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c3f1ac6..56cc251 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,13 @@ -use std::any::Any; +use std::convert::TryInto; +use std::str; -use scale_info::{Type, TypeDef}; +use scale_info::form::MetaForm; +use scale_info::{Type, TypeDef, TypeDefPrimitive}; +use serde::de::Visitor; +use serde::ser::SerializeSeq; +use serde::ser::SerializeTuple; use serde::ser::{Error, SerializeStruct}; -use serde::{Serialize, Serializer}; +use serde::{Deserialize, Serialize, Serializer}; pub struct Value<'a> { data: &'a [u8], @@ -21,22 +26,135 @@ impl<'a> Serialize for Value<'a> { S: serde::Serializer, { match self.info.type_def() { - TypeDef::Primitive(u32) => ser.serialize_u32(self.data[0].into()), + TypeDef::Primitive(primitive) => match primitive { + scale_info::TypeDefPrimitive::U8 => ser.serialize_u8(self.data[0].into()), + scale_info::TypeDefPrimitive::U16 => ser.serialize_u16(self.data[0].into()), + scale_info::TypeDefPrimitive::U32 => ser.serialize_u32(self.data[0].into()), + scale_info::TypeDefPrimitive::U64 => ser.serialize_u64(self.data[0].into()), + scale_info::TypeDefPrimitive::U128 => ser.serialize_u128(self.data[0].into()), + scale_info::TypeDefPrimitive::I8 => { + ser.serialize_i8(self.data[0].try_into().unwrap()) + } + scale_info::TypeDefPrimitive::I16 => { + ser.serialize_i16(self.data[0].try_into().unwrap()) + } + scale_info::TypeDefPrimitive::I32 => { + ser.serialize_i32(self.data[0].try_into().unwrap()) + } + scale_info::TypeDefPrimitive::I64 => { + ser.serialize_i64(self.data[0].try_into().unwrap()) + } + scale_info::TypeDefPrimitive::I128 => { + ser.serialize_i128(self.data[0].try_into().unwrap()) + } + scale_info::TypeDefPrimitive::Bool => ser.serialize_bool(self.data[0] != 0), + scale_info::TypeDefPrimitive::Char => ser.serialize_char(self.data[0].into()), + scale_info::TypeDefPrimitive::Str => { + ser.serialize_str(str::from_utf8(self.data).unwrap()) + } + _ => ser.serialize_bytes(self.data), + }, TypeDef::Composite(x) => { let fields = x.fields(); let mut state = ser.serialize_struct("", fields.len())?; - for (i, f) in fields.iter().enumerate() { - println!("{:?}", f); + let mut i = 0; + for (_, f) in fields.iter().enumerate() { let name = f.name().unwrap(); + let t = f.ty().type_info(); + let size = get_size(t.type_def()); state.serialize_field(name, &self.data[i])?; + i = i + size; } state.end() } - _ => ser.serialize_bytes(self.data), + TypeDef::Variant(_y) => ser.serialize_bytes(self.data), + TypeDef::Sequence(_seq) => { + let mut seq = ser.serialize_seq(Some(self.data.len()))?; + println!("{:?}", self.data); + for e in self.data { + seq.serialize_element(e)?; + } + seq.end() + } + TypeDef::Array(_) => { + let mut seq = ser.serialize_seq(Some(self.data.len()))?; + for e in self.data { + seq.serialize_element(e)?; + } + seq.end() + } + TypeDef::Tuple(x) => { + let mut seq = ser.serialize_tuple(x.fields().len())?; + let mut i = 0; + for (_, f) in x.fields().iter().enumerate() { + let size = get_size(f.type_info().type_def()); + seq.serialize_element(&self.data[i])?; + i = i + size; + } + seq.end() + } + TypeDef::Compact(_) => self.data.serialize(ser), + TypeDef::Phantom(_) => self.data.serialize(ser), } } } +fn get_size(t: &TypeDef) -> usize { + match t { + TypeDef::Primitive(primitive) => match primitive { + scale_info::TypeDefPrimitive::U8 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::U16 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::U32 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::U64 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::U128 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::I8 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::I16 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::I32 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::I64 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::I128 => std::mem::size_of::(), + scale_info::TypeDefPrimitive::Bool => std::mem::size_of::(), + scale_info::TypeDefPrimitive::Char => std::mem::size_of::(), + _ => 0, + }, + TypeDef::Composite(_) => todo!(), + TypeDef::Variant(_) => todo!(), + TypeDef::Sequence(_) => todo!(), + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(_) => todo!(), + TypeDef::Compact(_) => todo!(), + TypeDef::Phantom(_) => todo!(), + } +} + +impl<'de> Deserialize<'de> for Value<'de> { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct ValueVisitor; + + impl<'de> Visitor<'de> for ValueVisitor { + type Value = Value<'de>; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + todo!() + } + } + + deserializer.deserialize_u32(ValueVisitor) + + } + + fn deserialize_in_place(deserializer: D, place: &mut Self) -> Result<(), D::Error> + where + D: serde::Deserializer<'de>, + { + // Default implementation just delegates to `deserialize` impl. + *place = Deserialize::deserialize(deserializer)?; + Ok(()) + } +} + #[cfg(test)] mod tests { use std::error::Error; @@ -45,6 +163,7 @@ mod tests { use parity_scale_codec::Encode; use scale_info::TypeInfo; use serde_json::to_value; + use serde_json::from_slice; #[test] fn serialize_u32() -> Result<(), Box> { @@ -56,17 +175,74 @@ mod tests { Ok(()) } + #[test] + fn serialize_u64() -> Result<(), Box> { + let foo = 2u64; + let data = foo.encode(); + let info = u64::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_u16() -> Result<(), Box> { + let foo = 2u16; + let data = foo.encode(); + let info = u16::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_u8() -> Result<(), Box> { + let foo = 2u8; + let data = foo.encode(); + let info = u8::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_bool() -> Result<(), Box> { + let foo = true; + let data = foo.encode(); + let info = bool::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + // #[test] + // fn serialize_array() -> Result<(), Box> { + // let foo: Vec = [2u16, 7u16].into(); + // let data = foo.encode(); + // let info = Vec::::type_info(); + // let val = Value::new(&data, &info); + // assert_eq!(to_value(val)?, to_value(foo)?); + // Ok(()) + // } + + #[test] + fn serialize_tuple() -> Result<(), Box> { + let foo = (2u8, 2u8); + let data = foo.encode(); + let info = <(u8, u8)>::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + #[test] fn serialize_simple_struct() -> Result<(), Box> { #[derive(Encode, Serialize, TypeInfo)] struct Foo { bar: u32, - baz: bool, + baz: u32, } - let foo = Foo { - bar: 123, - baz: true, - }; + let foo = Foo { bar: 123, baz: 45 }; let data = foo.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); @@ -74,4 +250,15 @@ mod tests { assert_eq!(to_value(val)?, to_value(foo)?); Ok(()) } + + #[test] + fn deserialize_u32() -> Result<(), Box> { + let foo = 2u32; + let data = foo.encode(); + println!("{:?}", data); + let info = u32::type_info(); + let val = Value::new(&data, &info); + assert_eq!(from_slice::(val.data)?, from_slice::(&data)?); + Ok(()) + } } From a99f0cb49e0dc57735786596f02f31c3abfa19d3 Mon Sep 17 00:00:00 2001 From: Stanly Johnson Date: Mon, 23 Aug 2021 10:41:01 +0530 Subject: [PATCH 06/59] Remove deserialzie and lint fixes --- Cargo.toml | 2 +- src/lib.rs | 184 ++++++++++++++++++++++++++++++++--------------------- 2 files changed, 113 insertions(+), 73 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7887ce5..2ee3d54 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,9 @@ edition = "2018" [dependencies] scale-info = { version = "0.6.0", default-features = false } serde = { version = "1.0.126", default-features = false } +serde_json = "1.0.64" [dev-dependencies] parity-scale-codec = { version = "2.1.3", features = ["derive"] } scale-info = { version = "0.6.0", features = ["derive"] } serde = { version = "1.0.126", features = ["derive"] } -serde_json = "1.0.64" diff --git a/src/lib.rs b/src/lib.rs index 56cc251..d2ab0c0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,13 +1,12 @@ -use std::convert::TryInto; -use std::str; - use scale_info::form::MetaForm; -use scale_info::{Type, TypeDef, TypeDefPrimitive}; -use serde::de::Visitor; +use scale_info::{Type, TypeDef}; use serde::ser::SerializeSeq; -use serde::ser::SerializeTuple; -use serde::ser::{Error, SerializeStruct}; -use serde::{Deserialize, Serialize, Serializer}; +use serde::ser::{SerializeStruct, SerializeTuple}; +use serde::Serialize; +use serde_json::to_value; +use std::any::Any; +use std::convert::TryInto; +use std::str; pub struct Value<'a> { data: &'a [u8], @@ -27,7 +26,7 @@ impl<'a> Serialize for Value<'a> { { match self.info.type_def() { TypeDef::Primitive(primitive) => match primitive { - scale_info::TypeDefPrimitive::U8 => ser.serialize_u8(self.data[0].into()), + scale_info::TypeDefPrimitive::U8 => ser.serialize_u8(self.data[0]), scale_info::TypeDefPrimitive::U16 => ser.serialize_u16(self.data[0].into()), scale_info::TypeDefPrimitive::U32 => ser.serialize_u32(self.data[0].into()), scale_info::TypeDefPrimitive::U64 => ser.serialize_u64(self.data[0].into()), @@ -61,25 +60,45 @@ impl<'a> Serialize for Value<'a> { for (_, f) in fields.iter().enumerate() { let name = f.name().unwrap(); let t = f.ty().type_info(); - let size = get_size(t.type_def()); - state.serialize_field(name, &self.data[i])?; - i = i + size; + match t.type_def() { + TypeDef::Primitive(_) => { + let size = get_size(t.type_def()); + state.serialize_field(name, &self.data[i])?; + i += size; + }, + _ => { + println!("{:?}", t.type_def()); + println!("{:?}", self.data); + println!("{:?}", f.to_owned()); + let size = std::mem::size_of_val(&f.to_owned().ty()); + let u = &f.ty().type_info(); + let data = Value::new(&self.data, u); + println!("{:?}", size); + state.serialize_field(name, &data)?; + i += size; + } + } } state.end() } TypeDef::Variant(_y) => ser.serialize_bytes(self.data), - TypeDef::Sequence(_seq) => { + TypeDef::Sequence(x) => { + let size = get_size(x.type_param().type_info().type_def()); let mut seq = ser.serialize_seq(Some(self.data.len()))?; - println!("{:?}", self.data); - for e in self.data { - seq.serialize_element(e)?; + let mut i: usize = 1; + while i < self.data.len() { + seq.serialize_element(&self.data[i])?; + i += size; } seq.end() } - TypeDef::Array(_) => { + TypeDef::Array(x) => { + let size = get_size(x.type_param().type_info().type_def()); let mut seq = ser.serialize_seq(Some(self.data.len()))?; - for e in self.data { - seq.serialize_element(e)?; + let mut i: usize = 1; + while i < self.data.len() { + seq.serialize_element(&self.data[i])?; + i += size; } seq.end() } @@ -89,7 +108,7 @@ impl<'a> Serialize for Value<'a> { for (_, f) in x.fields().iter().enumerate() { let size = get_size(f.type_info().type_def()); seq.serialize_element(&self.data[i])?; - i = i + size; + i += size; } seq.end() } @@ -116,42 +135,7 @@ fn get_size(t: &TypeDef) -> usize { scale_info::TypeDefPrimitive::Char => std::mem::size_of::(), _ => 0, }, - TypeDef::Composite(_) => todo!(), - TypeDef::Variant(_) => todo!(), - TypeDef::Sequence(_) => todo!(), - TypeDef::Array(_) => todo!(), - TypeDef::Tuple(_) => todo!(), - TypeDef::Compact(_) => todo!(), - TypeDef::Phantom(_) => todo!(), - } -} - -impl<'de> Deserialize<'de> for Value<'de> { - fn deserialize(deserializer: D) -> Result - where - D: serde::Deserializer<'de>, - { - struct ValueVisitor; - - impl<'de> Visitor<'de> for ValueVisitor { - type Value = Value<'de>; - - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - todo!() - } - } - - deserializer.deserialize_u32(ValueVisitor) - - } - - fn deserialize_in_place(deserializer: D, place: &mut Self) -> Result<(), D::Error> - where - D: serde::Deserializer<'de>, - { - // Default implementation just delegates to `deserialize` impl. - *place = Deserialize::deserialize(deserializer)?; - Ok(()) + _ => 0, // ideally this match arm is not required } } @@ -163,7 +147,6 @@ mod tests { use parity_scale_codec::Encode; use scale_info::TypeInfo; use serde_json::to_value; - use serde_json::from_slice; #[test] fn serialize_u32() -> Result<(), Box> { @@ -215,15 +198,35 @@ mod tests { Ok(()) } - // #[test] - // fn serialize_array() -> Result<(), Box> { - // let foo: Vec = [2u16, 7u16].into(); - // let data = foo.encode(); - // let info = Vec::::type_info(); - // let val = Value::new(&data, &info); - // assert_eq!(to_value(val)?, to_value(foo)?); - // Ok(()) - // } + #[test] + fn serialize_u8array() -> Result<(), Box> { + let foo: Vec = [2u8, 7u8].into(); + let data = foo.encode(); + let info = Vec::::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_u16array() -> Result<(), Box> { + let foo: Vec = [2u16, 7u16].into(); + let data = foo.encode(); + let info = Vec::::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_u32array() -> Result<(), Box> { + let foo: Vec = [2u32, 7u32].into(); + let data = foo.encode(); + let info = Vec::::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } #[test] fn serialize_tuple() -> Result<(), Box> { @@ -236,7 +239,7 @@ mod tests { } #[test] - fn serialize_simple_struct() -> Result<(), Box> { + fn serialize_simple_u32struct() -> Result<(), Box> { #[derive(Encode, Serialize, TypeInfo)] struct Foo { bar: u32, @@ -252,13 +255,50 @@ mod tests { } #[test] - fn deserialize_u32() -> Result<(), Box> { - let foo = 2u32; + fn serialize_simple_u8struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: u8, + baz: u8, + } + let foo = Foo { bar: 123, baz: 45 }; let data = foo.encode(); - println!("{:?}", data); - let info = u32::type_info(); + let info = Foo::type_info(); + let val = Value::new(&data, &info); + + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_simple_u64struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: u64, + baz: u64, + } + let foo = Foo { bar: 123, baz: 45 }; + let data = foo.encode(); + let info = Foo::type_info(); let val = Value::new(&data, &info); - assert_eq!(from_slice::(val.data)?, from_slice::(&data)?); + + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_complex_struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: Vec, + baz: u8, + } + let foo = Foo { bar: [123u8].into(), baz: 45 }; + let data = foo.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, &info); + + assert_eq!(to_value(val)?, to_value(foo)?); Ok(()) } } From a53dd811f9969cdf3a8b4d8dc5982ec07d4cbd66 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Tue, 31 Aug 2021 17:51:09 +0200 Subject: [PATCH 07/59] Change tests to have more realistic values --- src/lib.rs | 70 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d2ab0c0..23a2136 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -65,18 +65,25 @@ impl<'a> Serialize for Value<'a> { let size = get_size(t.type_def()); state.serialize_field(name, &self.data[i])?; i += size; - }, + } _ => { println!("{:?}", t.type_def()); println!("{:?}", self.data); println!("{:?}", f.to_owned()); let size = std::mem::size_of_val(&f.to_owned().ty()); let u = &f.ty().type_info(); - let data = Value::new(&self.data, u); + let data = Value::new(&self.data, u); println!("{:?}", size); state.serialize_field(name, &data)?; i += size; } + TypeDef::Composite(_) => todo!(), + TypeDef::Variant(_) => todo!(), + TypeDef::Sequence(_) => todo!(), + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(_) => todo!(), + TypeDef::Compact(_) => todo!(), + TypeDef::Phantom(_) => todo!(), } } state.end() @@ -133,9 +140,17 @@ fn get_size(t: &TypeDef) -> usize { scale_info::TypeDefPrimitive::I128 => std::mem::size_of::(), scale_info::TypeDefPrimitive::Bool => std::mem::size_of::(), scale_info::TypeDefPrimitive::Char => std::mem::size_of::(), - _ => 0, + scale_info::TypeDefPrimitive::Str => todo!(), + scale_info::TypeDefPrimitive::U256 => todo!(), + scale_info::TypeDefPrimitive::I256 => todo!(), }, - _ => 0, // ideally this match arm is not required + TypeDef::Composite(_) => todo!(), + TypeDef::Variant(_) => todo!(), + TypeDef::Sequence(_) => todo!(), + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(_) => todo!(), + TypeDef::Compact(_) => todo!(), + TypeDef::Phantom(_) => todo!(), } } @@ -150,7 +165,7 @@ mod tests { #[test] fn serialize_u32() -> Result<(), Box> { - let foo = 2u32; + let foo = u32::MAX; let data = foo.encode(); let info = u32::type_info(); let val = Value::new(&data, &info); @@ -160,7 +175,7 @@ mod tests { #[test] fn serialize_u64() -> Result<(), Box> { - let foo = 2u64; + let foo = u64::MAX; let data = foo.encode(); let info = u64::type_info(); let val = Value::new(&data, &info); @@ -170,7 +185,7 @@ mod tests { #[test] fn serialize_u16() -> Result<(), Box> { - let foo = 2u16; + let foo = u16::MAX; let data = foo.encode(); let info = u16::type_info(); let val = Value::new(&data, &info); @@ -180,7 +195,7 @@ mod tests { #[test] fn serialize_u8() -> Result<(), Box> { - let foo = 2u8; + let foo = u8::MAX; let data = foo.encode(); let info = u8::type_info(); let val = Value::new(&data, &info); @@ -200,7 +215,7 @@ mod tests { #[test] fn serialize_u8array() -> Result<(), Box> { - let foo: Vec = [2u8, 7u8].into(); + let foo: Vec = [2u8, u8::MAX].into(); let data = foo.encode(); let info = Vec::::type_info(); let val = Value::new(&data, &info); @@ -210,7 +225,7 @@ mod tests { #[test] fn serialize_u16array() -> Result<(), Box> { - let foo: Vec = [2u16, 7u16].into(); + let foo: Vec = [2u16, u16::MAX].into(); let data = foo.encode(); let info = Vec::::type_info(); let val = Value::new(&data, &info); @@ -220,7 +235,7 @@ mod tests { #[test] fn serialize_u32array() -> Result<(), Box> { - let foo: Vec = [2u32, 7u32].into(); + let foo: Vec = [2u32, u32::MAX].into(); let data = foo.encode(); let info = Vec::::type_info(); let val = Value::new(&data, &info); @@ -230,9 +245,9 @@ mod tests { #[test] fn serialize_tuple() -> Result<(), Box> { - let foo = (2u8, 2u8); + let foo: (i128, Vec) = (i128::MIN, vec!["foo".into()]); let data = foo.encode(); - let info = <(u8, u8)>::type_info(); + let info = <(i128, Vec)>::type_info(); let val = Value::new(&data, &info); assert_eq!(to_value(val)?, to_value(foo)?); Ok(()) @@ -245,7 +260,10 @@ mod tests { bar: u32, baz: u32, } - let foo = Foo { bar: 123, baz: 45 }; + let foo = Foo { + bar: 123, + baz: u32::MAX, + }; let data = foo.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); @@ -261,7 +279,10 @@ mod tests { bar: u8, baz: u8, } - let foo = Foo { bar: 123, baz: 45 }; + let foo = Foo { + bar: 123, + baz: u8::MAX, + }; let data = foo.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); @@ -277,7 +298,10 @@ mod tests { bar: u64, baz: u64, } - let foo = Foo { bar: 123, baz: 45 }; + let foo = Foo { + bar: 123, + baz: u64::MAX, + }; let data = foo.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); @@ -288,12 +312,20 @@ mod tests { #[test] fn serialize_complex_struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + enum Bar { + This, + That(i16), + } #[derive(Encode, Serialize, TypeInfo)] struct Foo { - bar: Vec, - baz: u8, + bar: Vec, + baz: Option, } - let foo = Foo { bar: [123u8].into(), baz: 45 }; + let foo = Foo { + bar: [Bar::That(i16::MAX), Bar::This].into(), + baz: Some("aliquam malesuada bibendum arcu vitae".into()), + }; let data = foo.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); From b775a3f8be8cc371526e493cf8b3df5188b683db Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Tue, 31 Aug 2021 19:30:14 +0200 Subject: [PATCH 08/59] Fix primitive serialization --- Cargo.toml | 1 + src/lib.rs | 144 ++++++++++++++++++++++++++++++----------------------- 2 files changed, 84 insertions(+), 61 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 2ee3d54..5b35027 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ authors = ["Daniel Olano "] edition = "2018" [dependencies] +byteorder = { version = "1.4.3", default-features = false } scale-info = { version = "0.6.0", default-features = false } serde = { version = "1.0.126", default-features = false } serde_json = "1.0.64" diff --git a/src/lib.rs b/src/lib.rs index 23a2136..b22425a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,11 +1,10 @@ +use byteorder::{ByteOrder, LE}; use scale_info::form::MetaForm; +use scale_info::{prelude::*, TypeDefPrimitive}; use scale_info::{Type, TypeDef}; use serde::ser::SerializeSeq; use serde::ser::{SerializeStruct, SerializeTuple}; use serde::Serialize; -use serde_json::to_value; -use std::any::Any; -use std::convert::TryInto; use std::str; pub struct Value<'a> { @@ -24,33 +23,28 @@ impl<'a> Serialize for Value<'a> { where S: serde::Serializer, { - match self.info.type_def() { + let def = self.info.type_def(); + let size = size_of(def); + let data = self.data; + + match def { TypeDef::Primitive(primitive) => match primitive { - scale_info::TypeDefPrimitive::U8 => ser.serialize_u8(self.data[0]), - scale_info::TypeDefPrimitive::U16 => ser.serialize_u16(self.data[0].into()), - scale_info::TypeDefPrimitive::U32 => ser.serialize_u32(self.data[0].into()), - scale_info::TypeDefPrimitive::U64 => ser.serialize_u64(self.data[0].into()), - scale_info::TypeDefPrimitive::U128 => ser.serialize_u128(self.data[0].into()), - scale_info::TypeDefPrimitive::I8 => { - ser.serialize_i8(self.data[0].try_into().unwrap()) - } - scale_info::TypeDefPrimitive::I16 => { - ser.serialize_i16(self.data[0].try_into().unwrap()) - } - scale_info::TypeDefPrimitive::I32 => { - ser.serialize_i32(self.data[0].try_into().unwrap()) - } - scale_info::TypeDefPrimitive::I64 => { - ser.serialize_i64(self.data[0].try_into().unwrap()) - } - scale_info::TypeDefPrimitive::I128 => { - ser.serialize_i128(self.data[0].try_into().unwrap()) - } - scale_info::TypeDefPrimitive::Bool => ser.serialize_bool(self.data[0] != 0), - scale_info::TypeDefPrimitive::Char => ser.serialize_char(self.data[0].into()), - scale_info::TypeDefPrimitive::Str => { - ser.serialize_str(str::from_utf8(self.data).unwrap()) + TypeDefPrimitive::U8 => ser.serialize_u8(data[0]), + TypeDefPrimitive::U16 => ser.serialize_u16(LE::read_u16(&data[..size])), + TypeDefPrimitive::U32 => ser.serialize_u32(LE::read_u32(&data[..size])), + TypeDefPrimitive::U64 => ser.serialize_u64(LE::read_u64(&data[..size])), + TypeDefPrimitive::U128 => ser.serialize_u128(LE::read_u128(&data[..size])), + TypeDefPrimitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), + TypeDefPrimitive::I16 => ser.serialize_i16(LE::read_i16(&data[..size])), + TypeDefPrimitive::I32 => ser.serialize_i32(LE::read_i32(&data[..size])), + TypeDefPrimitive::I64 => ser.serialize_i64(LE::read_i64(&data[..size])), + TypeDefPrimitive::I128 => ser.serialize_i128(LE::read_i128(&data[..size])), + TypeDefPrimitive::Bool => ser.serialize_bool(data[0] != 0), + TypeDefPrimitive::Char => { + let n = LE::read_u32(&data[..size]); + ser.serialize_char(char::from_u32(n).unwrap()) } + TypeDefPrimitive::Str => ser.serialize_str(str::from_utf8(self.data).unwrap()), _ => ser.serialize_bytes(self.data), }, TypeDef::Composite(x) => { @@ -62,7 +56,7 @@ impl<'a> Serialize for Value<'a> { let t = f.ty().type_info(); match t.type_def() { TypeDef::Primitive(_) => { - let size = get_size(t.type_def()); + let size = size_of(t.type_def()); state.serialize_field(name, &self.data[i])?; i += size; } @@ -77,20 +71,20 @@ impl<'a> Serialize for Value<'a> { state.serialize_field(name, &data)?; i += size; } - TypeDef::Composite(_) => todo!(), - TypeDef::Variant(_) => todo!(), - TypeDef::Sequence(_) => todo!(), - TypeDef::Array(_) => todo!(), - TypeDef::Tuple(_) => todo!(), - TypeDef::Compact(_) => todo!(), - TypeDef::Phantom(_) => todo!(), + // TypeDef::Composite(_) => todo!(), + // TypeDef::Variant(_) => todo!(), + // TypeDef::Sequence(_) => todo!(), + // TypeDef::Array(_) => todo!(), + // TypeDef::Tuple(_) => todo!(), + // TypeDef::Compact(_) => todo!(), + // TypeDef::Phantom(_) => todo!(), } } state.end() } TypeDef::Variant(_y) => ser.serialize_bytes(self.data), TypeDef::Sequence(x) => { - let size = get_size(x.type_param().type_info().type_def()); + let size = size_of(x.type_param().type_info().type_def()); let mut seq = ser.serialize_seq(Some(self.data.len()))?; let mut i: usize = 1; while i < self.data.len() { @@ -100,7 +94,7 @@ impl<'a> Serialize for Value<'a> { seq.end() } TypeDef::Array(x) => { - let size = get_size(x.type_param().type_info().type_def()); + let size = size_of(x.type_param().type_info().type_def()); let mut seq = ser.serialize_seq(Some(self.data.len()))?; let mut i: usize = 1; while i < self.data.len() { @@ -113,7 +107,7 @@ impl<'a> Serialize for Value<'a> { let mut seq = ser.serialize_tuple(x.fields().len())?; let mut i = 0; for (_, f) in x.fields().iter().enumerate() { - let size = get_size(f.type_info().type_def()); + let size = size_of(f.type_info().type_def()); seq.serialize_element(&self.data[i])?; i += size; } @@ -125,24 +119,22 @@ impl<'a> Serialize for Value<'a> { } } -fn get_size(t: &TypeDef) -> usize { +fn size_of(t: &TypeDef) -> usize { match t { TypeDef::Primitive(primitive) => match primitive { - scale_info::TypeDefPrimitive::U8 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::U16 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::U32 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::U64 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::U128 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::I8 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::I16 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::I32 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::I64 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::I128 => std::mem::size_of::(), - scale_info::TypeDefPrimitive::Bool => std::mem::size_of::(), - scale_info::TypeDefPrimitive::Char => std::mem::size_of::(), - scale_info::TypeDefPrimitive::Str => todo!(), - scale_info::TypeDefPrimitive::U256 => todo!(), - scale_info::TypeDefPrimitive::I256 => todo!(), + scale_info::TypeDefPrimitive::U8 => mem::size_of::(), + scale_info::TypeDefPrimitive::U16 => mem::size_of::(), + scale_info::TypeDefPrimitive::U32 => mem::size_of::(), + scale_info::TypeDefPrimitive::U64 => mem::size_of::(), + scale_info::TypeDefPrimitive::U128 => mem::size_of::(), + scale_info::TypeDefPrimitive::I8 => mem::size_of::(), + scale_info::TypeDefPrimitive::I16 => mem::size_of::(), + scale_info::TypeDefPrimitive::I32 => mem::size_of::(), + scale_info::TypeDefPrimitive::I64 => mem::size_of::(), + scale_info::TypeDefPrimitive::I128 => mem::size_of::(), + scale_info::TypeDefPrimitive::Bool => mem::size_of::(), + scale_info::TypeDefPrimitive::Char => mem::size_of::(), + _ => unimplemented!(), }, TypeDef::Composite(_) => todo!(), TypeDef::Variant(_) => todo!(), @@ -163,6 +155,26 @@ mod tests { use scale_info::TypeInfo; use serde_json::to_value; + #[test] + fn serialize_u8() -> Result<(), Box> { + let foo = u8::MAX; + let data = foo.encode(); + let info = u8::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_u16() -> Result<(), Box> { + let foo = u16::MAX; + let data = foo.encode(); + let info = u16::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + #[test] fn serialize_u32() -> Result<(), Box> { let foo = u32::MAX; @@ -184,20 +196,30 @@ mod tests { } #[test] - fn serialize_u16() -> Result<(), Box> { - let foo = u16::MAX; + fn serialize_i16() -> Result<(), Box> { + let foo = i16::MAX; let data = foo.encode(); - let info = u16::type_info(); + let info = i16::type_info(); let val = Value::new(&data, &info); assert_eq!(to_value(val)?, to_value(foo)?); Ok(()) } #[test] - fn serialize_u8() -> Result<(), Box> { - let foo = u8::MAX; + fn serialize_i32() -> Result<(), Box> { + let foo = i32::MAX; let data = foo.encode(); - let info = u8::type_info(); + let info = i32::type_info(); + let val = Value::new(&data, &info); + assert_eq!(to_value(val)?, to_value(foo)?); + Ok(()) + } + + #[test] + fn serialize_i64() -> Result<(), Box> { + let foo = i64::MAX; + let data = foo.encode(); + let info = i64::type_info(); let val = Value::new(&data, &info); assert_eq!(to_value(val)?, to_value(foo)?); Ok(()) From 8e17aa50247721a3055cfee85ca2aaf686d7f4b4 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 1 Sep 2021 20:07:36 +0200 Subject: [PATCH 09/59] Setup serialization of composite types --- Cargo.toml | 8 +- src/lib.rs | 281 +++++++++++++++++++++++------------------------------ 2 files changed, 126 insertions(+), 163 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5b35027..36ce7a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,11 +6,11 @@ edition = "2018" [dependencies] byteorder = { version = "1.4.3", default-features = false } -scale-info = { version = "0.6.0", default-features = false } +scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } -serde_json = "1.0.64" +serde_json = "1.0.66" [dev-dependencies] -parity-scale-codec = { version = "2.1.3", features = ["derive"] } -scale-info = { version = "0.6.0", features = ["derive"] } +parity-scale-codec = { version = "2.2.0", features = ["derive"] } +scale-info = { version = "0.10.0", features = ["derive"] } serde = { version = "1.0.126", features = ["derive"] } diff --git a/src/lib.rs b/src/lib.rs index b22425a..05a900f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,12 +1,13 @@ use byteorder::{ByteOrder, LE}; -use scale_info::form::MetaForm; -use scale_info::{prelude::*, TypeDefPrimitive}; +use scale_info::{Field, TypeDefPrimitive as Primitive}; use scale_info::{Type, TypeDef}; -use serde::ser::SerializeSeq; -use serde::ser::{SerializeStruct, SerializeTuple}; +use serde::ser::{SerializeStruct, SerializeTupleStruct}; use serde::Serialize; use std::str; +/// A non-owning container for SCALE encoded data that can serialize types +/// directly with the help of a type registry and without using an intermediate +/// representation that requires allocating data. pub struct Value<'a> { data: &'a [u8], info: &'a Type, @@ -23,126 +24,88 @@ impl<'a> Serialize for Value<'a> { where S: serde::Serializer, { - let def = self.info.type_def(); - let size = size_of(def); let data = self.data; + let ty = self.info; + let name = ty_name(ty); - match def { - TypeDef::Primitive(primitive) => match primitive { - TypeDefPrimitive::U8 => ser.serialize_u8(data[0]), - TypeDefPrimitive::U16 => ser.serialize_u16(LE::read_u16(&data[..size])), - TypeDefPrimitive::U32 => ser.serialize_u32(LE::read_u32(&data[..size])), - TypeDefPrimitive::U64 => ser.serialize_u64(LE::read_u64(&data[..size])), - TypeDefPrimitive::U128 => ser.serialize_u128(LE::read_u128(&data[..size])), - TypeDefPrimitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), - TypeDefPrimitive::I16 => ser.serialize_i16(LE::read_i16(&data[..size])), - TypeDefPrimitive::I32 => ser.serialize_i32(LE::read_i32(&data[..size])), - TypeDefPrimitive::I64 => ser.serialize_i64(LE::read_i64(&data[..size])), - TypeDefPrimitive::I128 => ser.serialize_i128(LE::read_i128(&data[..size])), - TypeDefPrimitive::Bool => ser.serialize_bool(data[0] != 0), - TypeDefPrimitive::Char => { - let n = LE::read_u32(&data[..size]); - ser.serialize_char(char::from_u32(n).unwrap()) - } - TypeDefPrimitive::Str => ser.serialize_str(str::from_utf8(self.data).unwrap()), - _ => ser.serialize_bytes(self.data), - }, - TypeDef::Composite(x) => { - let fields = x.fields(); - let mut state = ser.serialize_struct("", fields.len())?; - let mut i = 0; - for (_, f) in fields.iter().enumerate() { - let name = f.name().unwrap(); - let t = f.ty().type_info(); - match t.type_def() { - TypeDef::Primitive(_) => { - let size = size_of(t.type_def()); - state.serialize_field(name, &self.data[i])?; - i += size; - } - _ => { - println!("{:?}", t.type_def()); - println!("{:?}", self.data); - println!("{:?}", f.to_owned()); - let size = std::mem::size_of_val(&f.to_owned().ty()); - let u = &f.ty().type_info(); - let data = Value::new(&self.data, u); - println!("{:?}", size); - state.serialize_field(name, &data)?; - i += size; - } - // TypeDef::Composite(_) => todo!(), - // TypeDef::Variant(_) => todo!(), - // TypeDef::Sequence(_) => todo!(), - // TypeDef::Array(_) => todo!(), - // TypeDef::Tuple(_) => todo!(), - // TypeDef::Compact(_) => todo!(), - // TypeDef::Phantom(_) => todo!(), + match ty.type_def() { + TypeDef::Composite(def) => { + let fields = def.fields(); + // Differentiating between a tuple struct and a normal one + if fields.first().and_then(Field::name).is_none() { + let state = ser.serialize_tuple_struct(name, fields.len())?; + // TODO!; + state.end() + } else { + let mut state = ser.serialize_struct(&name, fields.len())?; + for f in fields { + let v = extract_value(data, f.ty().type_info()); + state.serialize_field(f.name().unwrap(), &v)?; } + state.end() } - state.end() } - TypeDef::Variant(_y) => ser.serialize_bytes(self.data), - TypeDef::Sequence(x) => { - let size = size_of(x.type_param().type_info().type_def()); - let mut seq = ser.serialize_seq(Some(self.data.len()))?; - let mut i: usize = 1; - while i < self.data.len() { - seq.serialize_element(&self.data[i])?; - i += size; - } - seq.end() - } - TypeDef::Array(x) => { - let size = size_of(x.type_param().type_info().type_def()); - let mut seq = ser.serialize_seq(Some(self.data.len()))?; - let mut i: usize = 1; - while i < self.data.len() { - seq.serialize_element(&self.data[i])?; - i += size; + TypeDef::Variant(en) => { + for v in en.variants() { + if v.fields().is_empty() { + ser.serialize_unit_variant(name, v.index().into(), v.name())?; + todo!() + } else { + for _f in v.fields() { + todo!() + } + } } - seq.end() + todo!() } - TypeDef::Tuple(x) => { - let mut seq = ser.serialize_tuple(x.fields().len())?; - let mut i = 0; - for (_, f) in x.fields().iter().enumerate() { - let size = size_of(f.type_info().type_def()); - seq.serialize_element(&self.data[i])?; - i += size; + TypeDef::Sequence(_) => todo!(), + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(_) => todo!(), + TypeDef::Primitive(def) => { + match def { + Primitive::U8 => ser.serialize_u8(data[0]), + Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), + Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), + Primitive::U64 => ser.serialize_u64(LE::read_u64(data)), + Primitive::U128 => ser.serialize_u128(LE::read_u128(data)), + Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), + Primitive::I16 => ser.serialize_i16(LE::read_i16(data)), + Primitive::I32 => ser.serialize_i32(LE::read_i32(data)), + Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), + Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), + Primitive::Bool => ser.serialize_bool(data[0] != 0), + + // TODO: test if statements from here on actually work + Primitive::Char => { + let n = LE::read_u32(data); + ser.serialize_char(char::from_u32(n).unwrap()) + } + Primitive::Str => ser.serialize_str(str::from_utf8(data).unwrap()), + _ => ser.serialize_bytes(data), } - seq.end() } - TypeDef::Compact(_) => self.data.serialize(ser), - TypeDef::Phantom(_) => self.data.serialize(ser), + TypeDef::Compact(_) => todo!(), + TypeDef::BitSequence(_) => todo!(), } } } -fn size_of(t: &TypeDef) -> usize { - match t { - TypeDef::Primitive(primitive) => match primitive { - scale_info::TypeDefPrimitive::U8 => mem::size_of::(), - scale_info::TypeDefPrimitive::U16 => mem::size_of::(), - scale_info::TypeDefPrimitive::U32 => mem::size_of::(), - scale_info::TypeDefPrimitive::U64 => mem::size_of::(), - scale_info::TypeDefPrimitive::U128 => mem::size_of::(), - scale_info::TypeDefPrimitive::I8 => mem::size_of::(), - scale_info::TypeDefPrimitive::I16 => mem::size_of::(), - scale_info::TypeDefPrimitive::I32 => mem::size_of::(), - scale_info::TypeDefPrimitive::I64 => mem::size_of::(), - scale_info::TypeDefPrimitive::I128 => mem::size_of::(), - scale_info::TypeDefPrimitive::Bool => mem::size_of::(), - scale_info::TypeDefPrimitive::Char => mem::size_of::(), - _ => unimplemented!(), - }, - TypeDef::Composite(_) => todo!(), - TypeDef::Variant(_) => todo!(), - TypeDef::Sequence(_) => todo!(), - TypeDef::Array(_) => todo!(), - TypeDef::Tuple(_) => todo!(), - TypeDef::Compact(_) => todo!(), - TypeDef::Phantom(_) => todo!(), +fn ty_name(ty: &Type) -> &'static str { + ty.path().segments().last().copied().unwrap_or("") +} + +// (TODO) Construct a Value slicing +fn extract_value(_data: &[u8], ty: Type) -> Value { + use TypeDef::*; + match ty.type_def() { + Composite(_def) => todo!(), + Variant(_def) => todo!(), + Sequence(_def) => todo!(), + Array(_def) => todo!(), + Tuple(_def) => todo!(), + Primitive(_def) => todo!(), + Compact(_def) => todo!(), + BitSequence(_def) => todo!(), } } @@ -157,121 +120,121 @@ mod tests { #[test] fn serialize_u8() -> Result<(), Box> { - let foo = u8::MAX; - let data = foo.encode(); + let extract_value = u8::MAX; + let data = extract_value.encode(); let info = u8::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_u16() -> Result<(), Box> { - let foo = u16::MAX; - let data = foo.encode(); + let extract_value = u16::MAX; + let data = extract_value.encode(); let info = u16::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_u32() -> Result<(), Box> { - let foo = u32::MAX; - let data = foo.encode(); + let extract_value = u32::MAX; + let data = extract_value.encode(); let info = u32::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_u64() -> Result<(), Box> { - let foo = u64::MAX; - let data = foo.encode(); + let extract_value = u64::MAX; + let data = extract_value.encode(); let info = u64::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_i16() -> Result<(), Box> { - let foo = i16::MAX; - let data = foo.encode(); + let extract_value = i16::MAX; + let data = extract_value.encode(); let info = i16::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_i32() -> Result<(), Box> { - let foo = i32::MAX; - let data = foo.encode(); + let extract_value = i32::MAX; + let data = extract_value.encode(); let info = i32::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_i64() -> Result<(), Box> { - let foo = i64::MAX; - let data = foo.encode(); + let extract_value = i64::MAX; + let data = extract_value.encode(); let info = i64::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_bool() -> Result<(), Box> { - let foo = true; - let data = foo.encode(); + let extract_value = true; + let data = extract_value.encode(); let info = bool::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_u8array() -> Result<(), Box> { - let foo: Vec = [2u8, u8::MAX].into(); - let data = foo.encode(); + let extract_value: Vec = [2u8, u8::MAX].into(); + let data = extract_value.encode(); let info = Vec::::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_u16array() -> Result<(), Box> { - let foo: Vec = [2u16, u16::MAX].into(); - let data = foo.encode(); + let extract_value: Vec = [2u16, u16::MAX].into(); + let data = extract_value.encode(); let info = Vec::::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_u32array() -> Result<(), Box> { - let foo: Vec = [2u32, u32::MAX].into(); - let data = foo.encode(); + let extract_value: Vec = [2u32, u32::MAX].into(); + let data = extract_value.encode(); let info = Vec::::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } #[test] fn serialize_tuple() -> Result<(), Box> { - let foo: (i128, Vec) = (i128::MIN, vec!["foo".into()]); - let data = foo.encode(); + let extract_value: (i128, Vec) = (i128::MIN, vec!["extract_value".into()]); + let data = extract_value.encode(); let info = <(i128, Vec)>::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -282,15 +245,15 @@ mod tests { bar: u32, baz: u32, } - let foo = Foo { + let extract_value = Foo { bar: 123, baz: u32::MAX, }; - let data = foo.encode(); + let data = extract_value.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -301,15 +264,15 @@ mod tests { bar: u8, baz: u8, } - let foo = Foo { + let extract_value = Foo { bar: 123, baz: u8::MAX, }; - let data = foo.encode(); + let data = extract_value.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -320,15 +283,15 @@ mod tests { bar: u64, baz: u64, } - let foo = Foo { + let extract_value = Foo { bar: 123, baz: u64::MAX, }; - let data = foo.encode(); + let data = extract_value.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -344,15 +307,15 @@ mod tests { bar: Vec, baz: Option, } - let foo = Foo { + let extract_value = Foo { bar: [Bar::That(i16::MAX), Bar::This].into(), baz: Some("aliquam malesuada bibendum arcu vitae".into()), }; - let data = foo.encode(); + let data = extract_value.encode(); let info = Foo::type_info(); let val = Value::new(&data, &info); - assert_eq!(to_value(val)?, to_value(foo)?); + assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } } From 8b985e8ec866e5015f903f5814a4c37fcabafd6f Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 2 Sep 2021 11:34:06 +0200 Subject: [PATCH 10/59] Make Value own the type data --- src/lib.rs | 87 ++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 58 insertions(+), 29 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 05a900f..abd52e4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,5 @@ use byteorder::{ByteOrder, LE}; -use scale_info::{Field, TypeDefPrimitive as Primitive}; -use scale_info::{Type, TypeDef}; +use scale_info::{prelude::*, Field, Type, TypeDef, TypeDefPrimitive as Primitive}; use serde::ser::{SerializeStruct, SerializeTupleStruct}; use serde::Serialize; use std::str; @@ -10,12 +9,12 @@ use std::str; /// representation that requires allocating data. pub struct Value<'a> { data: &'a [u8], - info: &'a Type, + ty: Type, } impl<'a> Value<'a> { - pub fn new(data: &'a [u8], info: &'a Type) -> Self { - Value { data, info } + pub fn new(data: &'a [u8], ty: Type) -> Self { + Value { data, ty } } } @@ -24,11 +23,10 @@ impl<'a> Serialize for Value<'a> { where S: serde::Serializer, { - let data = self.data; - let ty = self.info; - let name = ty_name(ty); + let mut data = self.data; + let name = ty_name(&self.ty); - match ty.type_def() { + match self.ty.type_def() { TypeDef::Composite(def) => { let fields = def.fields(); // Differentiating between a tuple struct and a normal one @@ -39,7 +37,7 @@ impl<'a> Serialize for Value<'a> { } else { let mut state = ser.serialize_struct(&name, fields.len())?; for f in fields { - let v = extract_value(data, f.ty().type_info()); + let v = extract_value(&mut data, f.ty().type_info()); state.serialize_field(f.name().unwrap(), &v)?; } state.end() @@ -94,8 +92,35 @@ fn ty_name(ty: &Type) -> &'static str { ty.path().segments().last().copied().unwrap_or("") } -// (TODO) Construct a Value slicing -fn extract_value(_data: &[u8], ty: Type) -> Value { +fn size_of(t: &Type) -> usize { + match t.type_def() { + TypeDef::Primitive(primitive) => match primitive { + scale_info::TypeDefPrimitive::U8 => mem::size_of::(), + scale_info::TypeDefPrimitive::U16 => mem::size_of::(), + scale_info::TypeDefPrimitive::U32 => mem::size_of::(), + scale_info::TypeDefPrimitive::U64 => mem::size_of::(), + scale_info::TypeDefPrimitive::U128 => mem::size_of::(), + scale_info::TypeDefPrimitive::I8 => mem::size_of::(), + scale_info::TypeDefPrimitive::I16 => mem::size_of::(), + scale_info::TypeDefPrimitive::I32 => mem::size_of::(), + scale_info::TypeDefPrimitive::I64 => mem::size_of::(), + scale_info::TypeDefPrimitive::I128 => mem::size_of::(), + scale_info::TypeDefPrimitive::Bool => mem::size_of::(), + scale_info::TypeDefPrimitive::Char => mem::size_of::(), + _ => unimplemented!(), + }, + TypeDef::Composite(_) => todo!(), + TypeDef::Variant(_) => todo!(), + TypeDef::Sequence(_) => todo!(), + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(_) => todo!(), + TypeDef::Compact(_) => todo!(), + TypeDef::BitSequence(_) => todo!(), + } +} + +// (TODO) Construct a Value consuming the original slice +fn extract_value<'a>(data: &mut &'a [u8], ty: Type) -> Value<'a> { use TypeDef::*; match ty.type_def() { Composite(_def) => todo!(), @@ -103,7 +128,11 @@ fn extract_value(_data: &[u8], ty: Type) -> Value { Sequence(_def) => todo!(), Array(_def) => todo!(), Tuple(_def) => todo!(), - Primitive(_def) => todo!(), + Primitive(_def) => { + let (value, reminder) = data.split_at(size_of(&ty)); + *data = reminder; + Value::new(value, ty) + } Compact(_def) => todo!(), BitSequence(_def) => todo!(), } @@ -123,7 +152,7 @@ mod tests { let extract_value = u8::MAX; let data = extract_value.encode(); let info = u8::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -133,7 +162,7 @@ mod tests { let extract_value = u16::MAX; let data = extract_value.encode(); let info = u16::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -143,7 +172,7 @@ mod tests { let extract_value = u32::MAX; let data = extract_value.encode(); let info = u32::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -153,7 +182,7 @@ mod tests { let extract_value = u64::MAX; let data = extract_value.encode(); let info = u64::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -163,7 +192,7 @@ mod tests { let extract_value = i16::MAX; let data = extract_value.encode(); let info = i16::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -173,7 +202,7 @@ mod tests { let extract_value = i32::MAX; let data = extract_value.encode(); let info = i32::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -183,7 +212,7 @@ mod tests { let extract_value = i64::MAX; let data = extract_value.encode(); let info = i64::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -193,7 +222,7 @@ mod tests { let extract_value = true; let data = extract_value.encode(); let info = bool::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -203,7 +232,7 @@ mod tests { let extract_value: Vec = [2u8, u8::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -213,7 +242,7 @@ mod tests { let extract_value: Vec = [2u16, u16::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -223,7 +252,7 @@ mod tests { let extract_value: Vec = [2u32, u32::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -233,7 +262,7 @@ mod tests { let extract_value: (i128, Vec) = (i128::MIN, vec!["extract_value".into()]); let data = extract_value.encode(); let info = <(i128, Vec)>::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -251,7 +280,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) @@ -270,7 +299,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) @@ -289,7 +318,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) @@ -313,7 +342,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, &info); + let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) From 2887915a9c270b49ee1cfc08d822c0d3a473ec88 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 3 Sep 2021 11:50:11 +0200 Subject: [PATCH 11/59] Serialize tuples and sequences --- src/lib.rs | 129 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 46 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index abd52e4..0b143bb 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,8 @@ use byteorder::{ByteOrder, LE}; use scale_info::{prelude::*, Field, Type, TypeDef, TypeDefPrimitive as Primitive}; -use serde::ser::{SerializeStruct, SerializeTupleStruct}; +use serde::ser::{SerializeSeq, SerializeStruct, SerializeTuple, SerializeTupleStruct}; use serde::Serialize; +use std::convert::TryInto; use std::str; /// A non-owning container for SCALE encoded data that can serialize types @@ -27,8 +28,8 @@ impl<'a> Serialize for Value<'a> { let name = ty_name(&self.ty); match self.ty.type_def() { - TypeDef::Composite(def) => { - let fields = def.fields(); + TypeDef::Composite(cmp) => { + let fields = cmp.fields(); // Differentiating between a tuple struct and a normal one if fields.first().and_then(Field::name).is_none() { let state = ser.serialize_tuple_struct(name, fields.len())?; @@ -37,14 +38,17 @@ impl<'a> Serialize for Value<'a> { } else { let mut state = ser.serialize_struct(&name, fields.len())?; for f in fields { - let v = extract_value(&mut data, f.ty().type_info()); - state.serialize_field(f.name().unwrap(), &v)?; + let ty = f.ty().type_info(); + let size = type_byte_count(&ty, data); + let (val, reminder) = data.split_at(size); + data = reminder; + state.serialize_field(f.name().unwrap(), &Value::new(val, ty))?; } state.end() } } - TypeDef::Variant(en) => { - for v in en.variants() { + TypeDef::Variant(enu) => { + for v in enu.variants() { if v.fields().is_empty() { ser.serialize_unit_variant(name, v.index().into(), v.name())?; todo!() @@ -56,11 +60,32 @@ impl<'a> Serialize for Value<'a> { } todo!() } - TypeDef::Sequence(_) => todo!(), + TypeDef::Sequence(seq) => { + let ty = seq.type_param().type_info(); + let (len, p_size) = sequence_len(data); + let mut data = &data[p_size..]; + let mut seq = ser.serialize_seq(Some(len))?; + + for _ in 0..len { + let ts = type_byte_count(&ty, data); + seq.serialize_element(&Value::new(&data[..ts], ty.clone()))?; + data = &data[ts..]; + } + seq.end() + } TypeDef::Array(_) => todo!(), - TypeDef::Tuple(_) => todo!(), - TypeDef::Primitive(def) => { - match def { + TypeDef::Tuple(tup) => { + let mut state = ser.serialize_tuple(tup.fields().len())?; + for f in tup.fields() { + let ty = f.type_info(); + let (val, reminder) = data.split_at(type_byte_count(&ty, data)); + data = reminder; + state.serialize_element(&Value::new(val, ty))?; + } + state.end() + } + TypeDef::Primitive(prm) => { + match prm { Primitive::U8 => ser.serialize_u8(data[0]), Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), @@ -72,13 +97,15 @@ impl<'a> Serialize for Value<'a> { Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), Primitive::Bool => ser.serialize_bool(data[0] != 0), - + Primitive::Str => { + let (_, s) = sequence_len(data); + ser.serialize_str(str::from_utf8(&data[s..]).unwrap()) + } // TODO: test if statements from here on actually work Primitive::Char => { let n = LE::read_u32(data); ser.serialize_char(char::from_u32(n).unwrap()) } - Primitive::Str => ser.serialize_str(str::from_utf8(data).unwrap()), _ => ser.serialize_bytes(data), } } @@ -92,26 +119,34 @@ fn ty_name(ty: &Type) -> &'static str { ty.path().segments().last().copied().unwrap_or("") } -fn size_of(t: &Type) -> usize { +fn type_byte_count<'a>(t: &Type, data: &'a [u8]) -> usize { match t.type_def() { - TypeDef::Primitive(primitive) => match primitive { - scale_info::TypeDefPrimitive::U8 => mem::size_of::(), - scale_info::TypeDefPrimitive::U16 => mem::size_of::(), - scale_info::TypeDefPrimitive::U32 => mem::size_of::(), - scale_info::TypeDefPrimitive::U64 => mem::size_of::(), - scale_info::TypeDefPrimitive::U128 => mem::size_of::(), - scale_info::TypeDefPrimitive::I8 => mem::size_of::(), - scale_info::TypeDefPrimitive::I16 => mem::size_of::(), - scale_info::TypeDefPrimitive::I32 => mem::size_of::(), - scale_info::TypeDefPrimitive::I64 => mem::size_of::(), - scale_info::TypeDefPrimitive::I128 => mem::size_of::(), - scale_info::TypeDefPrimitive::Bool => mem::size_of::(), - scale_info::TypeDefPrimitive::Char => mem::size_of::(), + TypeDef::Primitive(p) => match p { + Primitive::U8 => mem::size_of::(), + Primitive::U16 => mem::size_of::(), + Primitive::U32 => mem::size_of::(), + Primitive::U64 => mem::size_of::(), + Primitive::U128 => mem::size_of::(), + Primitive::I8 => mem::size_of::(), + Primitive::I16 => mem::size_of::(), + Primitive::I32 => mem::size_of::(), + Primitive::I64 => mem::size_of::(), + Primitive::I128 => mem::size_of::(), + Primitive::Bool => mem::size_of::(), + Primitive::Char => mem::size_of::(), + Primitive::Str => { + let (l, p_size) = sequence_len(data); + l + p_size + } _ => unimplemented!(), }, TypeDef::Composite(_) => todo!(), TypeDef::Variant(_) => todo!(), - TypeDef::Sequence(_) => todo!(), + TypeDef::Sequence(s) => { + let (len, prefix_size) = sequence_len(data); + let ty = s.type_param().type_info(); + (0..len).fold(prefix_size, |c, _| c + type_byte_count(&ty, &data[c..])) + } TypeDef::Array(_) => todo!(), TypeDef::Tuple(_) => todo!(), TypeDef::Compact(_) => todo!(), @@ -119,22 +154,20 @@ fn size_of(t: &Type) -> usize { } } -// (TODO) Construct a Value consuming the original slice -fn extract_value<'a>(data: &mut &'a [u8], ty: Type) -> Value<'a> { - use TypeDef::*; - match ty.type_def() { - Composite(_def) => todo!(), - Variant(_def) => todo!(), - Sequence(_def) => todo!(), - Array(_def) => todo!(), - Tuple(_def) => todo!(), - Primitive(_def) => { - let (value, reminder) = data.split_at(size_of(&ty)); - *data = reminder; - Value::new(value, ty) - } - Compact(_def) => todo!(), - BitSequence(_def) => todo!(), +fn sequence_len(data: &[u8]) -> (usize, usize) { + // need to peek at the data to know the length of sequence + // first byte(s) gives us a hint of the(compact encoded) length + // https://substrate.dev/docs/en/knowledgebase/advanced/codec#compactgeneral-integers + match data[0] % 0b100 { + 0 => ((data[0] >> 2).into(), 1), + 1 => (u16::from_le_bytes([(data[0] >> 2), data[1]]).into(), 2), + 2 => ( + u32::from_le_bytes([(data[0] >> 2), data[1], data[2], data[3]]) + .try_into() + .unwrap(), + 4, + ), + _ => todo!(), } } @@ -259,9 +292,13 @@ mod tests { #[test] fn serialize_tuple() -> Result<(), Box> { - let extract_value: (i128, Vec) = (i128::MIN, vec!["extract_value".into()]); + let extract_value: (i64, Vec, bool) = ( + i64::MIN, + vec!["hello".into(), "big".into(), "world".into()], + true, + ); let data = extract_value.encode(); - let info = <(i128, Vec)>::type_info(); + let info = <(i64, Vec, bool)>::type_info(); let val = Value::new(&data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) From cf920c97debb8d62f5b3d3b836c16a4c6fd5bcf6 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 3 Sep 2021 18:33:07 +0200 Subject: [PATCH 12/59] Serialize enums --- Cargo.toml | 2 +- src/lib.rs | 392 +------------------------------------------- src/value.rs | 447 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 454 insertions(+), 387 deletions(-) create mode 100644 src/value.rs diff --git a/Cargo.toml b/Cargo.toml index 36ce7a2..3ce969f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,9 +8,9 @@ edition = "2018" byteorder = { version = "1.4.3", default-features = false } scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } -serde_json = "1.0.66" [dev-dependencies] parity-scale-codec = { version = "2.2.0", features = ["derive"] } scale-info = { version = "0.10.0", features = ["derive"] } serde = { version = "1.0.126", features = ["derive"] } +serde_json = "1.0.66" diff --git a/src/lib.rs b/src/lib.rs index 0b143bb..55ffc7f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,387 +1,7 @@ -use byteorder::{ByteOrder, LE}; -use scale_info::{prelude::*, Field, Type, TypeDef, TypeDefPrimitive as Primitive}; -use serde::ser::{SerializeSeq, SerializeStruct, SerializeTuple, SerializeTupleStruct}; -use serde::Serialize; -use std::convert::TryInto; -use std::str; +///! # Scales +///! +///! Dynamic SCALE Serialization using `scale-info` type information. +///! +mod value; -/// A non-owning container for SCALE encoded data that can serialize types -/// directly with the help of a type registry and without using an intermediate -/// representation that requires allocating data. -pub struct Value<'a> { - data: &'a [u8], - ty: Type, -} - -impl<'a> Value<'a> { - pub fn new(data: &'a [u8], ty: Type) -> Self { - Value { data, ty } - } -} - -impl<'a> Serialize for Value<'a> { - fn serialize(&self, ser: S) -> Result - where - S: serde::Serializer, - { - let mut data = self.data; - let name = ty_name(&self.ty); - - match self.ty.type_def() { - TypeDef::Composite(cmp) => { - let fields = cmp.fields(); - // Differentiating between a tuple struct and a normal one - if fields.first().and_then(Field::name).is_none() { - let state = ser.serialize_tuple_struct(name, fields.len())?; - // TODO!; - state.end() - } else { - let mut state = ser.serialize_struct(&name, fields.len())?; - for f in fields { - let ty = f.ty().type_info(); - let size = type_byte_count(&ty, data); - let (val, reminder) = data.split_at(size); - data = reminder; - state.serialize_field(f.name().unwrap(), &Value::new(val, ty))?; - } - state.end() - } - } - TypeDef::Variant(enu) => { - for v in enu.variants() { - if v.fields().is_empty() { - ser.serialize_unit_variant(name, v.index().into(), v.name())?; - todo!() - } else { - for _f in v.fields() { - todo!() - } - } - } - todo!() - } - TypeDef::Sequence(seq) => { - let ty = seq.type_param().type_info(); - let (len, p_size) = sequence_len(data); - let mut data = &data[p_size..]; - let mut seq = ser.serialize_seq(Some(len))?; - - for _ in 0..len { - let ts = type_byte_count(&ty, data); - seq.serialize_element(&Value::new(&data[..ts], ty.clone()))?; - data = &data[ts..]; - } - seq.end() - } - TypeDef::Array(_) => todo!(), - TypeDef::Tuple(tup) => { - let mut state = ser.serialize_tuple(tup.fields().len())?; - for f in tup.fields() { - let ty = f.type_info(); - let (val, reminder) = data.split_at(type_byte_count(&ty, data)); - data = reminder; - state.serialize_element(&Value::new(val, ty))?; - } - state.end() - } - TypeDef::Primitive(prm) => { - match prm { - Primitive::U8 => ser.serialize_u8(data[0]), - Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), - Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), - Primitive::U64 => ser.serialize_u64(LE::read_u64(data)), - Primitive::U128 => ser.serialize_u128(LE::read_u128(data)), - Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), - Primitive::I16 => ser.serialize_i16(LE::read_i16(data)), - Primitive::I32 => ser.serialize_i32(LE::read_i32(data)), - Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), - Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), - Primitive::Bool => ser.serialize_bool(data[0] != 0), - Primitive::Str => { - let (_, s) = sequence_len(data); - ser.serialize_str(str::from_utf8(&data[s..]).unwrap()) - } - // TODO: test if statements from here on actually work - Primitive::Char => { - let n = LE::read_u32(data); - ser.serialize_char(char::from_u32(n).unwrap()) - } - _ => ser.serialize_bytes(data), - } - } - TypeDef::Compact(_) => todo!(), - TypeDef::BitSequence(_) => todo!(), - } - } -} - -fn ty_name(ty: &Type) -> &'static str { - ty.path().segments().last().copied().unwrap_or("") -} - -fn type_byte_count<'a>(t: &Type, data: &'a [u8]) -> usize { - match t.type_def() { - TypeDef::Primitive(p) => match p { - Primitive::U8 => mem::size_of::(), - Primitive::U16 => mem::size_of::(), - Primitive::U32 => mem::size_of::(), - Primitive::U64 => mem::size_of::(), - Primitive::U128 => mem::size_of::(), - Primitive::I8 => mem::size_of::(), - Primitive::I16 => mem::size_of::(), - Primitive::I32 => mem::size_of::(), - Primitive::I64 => mem::size_of::(), - Primitive::I128 => mem::size_of::(), - Primitive::Bool => mem::size_of::(), - Primitive::Char => mem::size_of::(), - Primitive::Str => { - let (l, p_size) = sequence_len(data); - l + p_size - } - _ => unimplemented!(), - }, - TypeDef::Composite(_) => todo!(), - TypeDef::Variant(_) => todo!(), - TypeDef::Sequence(s) => { - let (len, prefix_size) = sequence_len(data); - let ty = s.type_param().type_info(); - (0..len).fold(prefix_size, |c, _| c + type_byte_count(&ty, &data[c..])) - } - TypeDef::Array(_) => todo!(), - TypeDef::Tuple(_) => todo!(), - TypeDef::Compact(_) => todo!(), - TypeDef::BitSequence(_) => todo!(), - } -} - -fn sequence_len(data: &[u8]) -> (usize, usize) { - // need to peek at the data to know the length of sequence - // first byte(s) gives us a hint of the(compact encoded) length - // https://substrate.dev/docs/en/knowledgebase/advanced/codec#compactgeneral-integers - match data[0] % 0b100 { - 0 => ((data[0] >> 2).into(), 1), - 1 => (u16::from_le_bytes([(data[0] >> 2), data[1]]).into(), 2), - 2 => ( - u32::from_le_bytes([(data[0] >> 2), data[1], data[2], data[3]]) - .try_into() - .unwrap(), - 4, - ), - _ => todo!(), - } -} - -#[cfg(test)] -mod tests { - use std::error::Error; - - use super::*; - use parity_scale_codec::Encode; - use scale_info::TypeInfo; - use serde_json::to_value; - - #[test] - fn serialize_u8() -> Result<(), Box> { - let extract_value = u8::MAX; - let data = extract_value.encode(); - let info = u8::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_u16() -> Result<(), Box> { - let extract_value = u16::MAX; - let data = extract_value.encode(); - let info = u16::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_u32() -> Result<(), Box> { - let extract_value = u32::MAX; - let data = extract_value.encode(); - let info = u32::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_u64() -> Result<(), Box> { - let extract_value = u64::MAX; - let data = extract_value.encode(); - let info = u64::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_i16() -> Result<(), Box> { - let extract_value = i16::MAX; - let data = extract_value.encode(); - let info = i16::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_i32() -> Result<(), Box> { - let extract_value = i32::MAX; - let data = extract_value.encode(); - let info = i32::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_i64() -> Result<(), Box> { - let extract_value = i64::MAX; - let data = extract_value.encode(); - let info = i64::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_bool() -> Result<(), Box> { - let extract_value = true; - let data = extract_value.encode(); - let info = bool::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_u8array() -> Result<(), Box> { - let extract_value: Vec = [2u8, u8::MAX].into(); - let data = extract_value.encode(); - let info = Vec::::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_u16array() -> Result<(), Box> { - let extract_value: Vec = [2u16, u16::MAX].into(); - let data = extract_value.encode(); - let info = Vec::::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_u32array() -> Result<(), Box> { - let extract_value: Vec = [2u32, u32::MAX].into(); - let data = extract_value.encode(); - let info = Vec::::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_tuple() -> Result<(), Box> { - let extract_value: (i64, Vec, bool) = ( - i64::MIN, - vec!["hello".into(), "big".into(), "world".into()], - true, - ); - let data = extract_value.encode(); - let info = <(i64, Vec, bool)>::type_info(); - let val = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_simple_u32struct() -> Result<(), Box> { - #[derive(Encode, Serialize, TypeInfo)] - struct Foo { - bar: u32, - baz: u32, - } - let extract_value = Foo { - bar: 123, - baz: u32::MAX, - }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(&data, info); - - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_simple_u8struct() -> Result<(), Box> { - #[derive(Encode, Serialize, TypeInfo)] - struct Foo { - bar: u8, - baz: u8, - } - let extract_value = Foo { - bar: 123, - baz: u8::MAX, - }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(&data, info); - - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_simple_u64struct() -> Result<(), Box> { - #[derive(Encode, Serialize, TypeInfo)] - struct Foo { - bar: u64, - baz: u64, - } - let extract_value = Foo { - bar: 123, - baz: u64::MAX, - }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(&data, info); - - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } - - #[test] - fn serialize_complex_struct() -> Result<(), Box> { - #[derive(Encode, Serialize, TypeInfo)] - enum Bar { - This, - That(i16), - } - #[derive(Encode, Serialize, TypeInfo)] - struct Foo { - bar: Vec, - baz: Option, - } - let extract_value = Foo { - bar: [Bar::That(i16::MAX), Bar::This].into(), - baz: Some("aliquam malesuada bibendum arcu vitae".into()), - }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(&data, info); - - assert_eq!(to_value(val)?, to_value(extract_value)?); - Ok(()) - } -} +pub use value::Value; diff --git a/src/value.rs b/src/value.rs new file mode 100644 index 0000000..39781aa --- /dev/null +++ b/src/value.rs @@ -0,0 +1,447 @@ +use byteorder::{ByteOrder, LE}; +use scale_info::{prelude::*, Field, Type, TypeDef, TypeDefPrimitive as Primitive}; +use serde::ser::{ + SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, SerializeTupleStruct, + SerializeTupleVariant, +}; +use serde::Serialize; +use std::convert::TryInto; +use std::str; + +/// A non-owning container for SCALE encoded data that can serialize types +/// directly with the help of a type registry and without using an intermediate +/// representation that requires allocating data. +pub struct Value<'a> { + data: &'a [u8], + ty: Type, +} + +impl<'a> Value<'a> { + pub fn new(data: &'a [u8], ty: Type) -> Self { + Value { data, ty } + } +} + +impl<'a> Serialize for Value<'a> { + fn serialize(&self, ser: S) -> Result + where + S: serde::Serializer, + { + let mut data = self.data; + let name = ty_name(&self.ty); + + match self.ty.type_def() { + TypeDef::Composite(cmp) => { + let fields = cmp.fields(); + if is_tuple(fields) { + let state = ser.serialize_tuple_struct(name, fields.len())?; + // TODO!; + state.end() + } else { + let mut state = ser.serialize_struct(&name, fields.len())?; + for f in fields { + let ty = f.ty().type_info(); + let size = ty_data_size(&ty, data); + let (val, reminder) = data.split_at(size); + data = reminder; + state.serialize_field(f.name().unwrap(), &Value::new(val, ty))?; + } + state.end() + } + } + TypeDef::Variant(enu) => { + let (idx, d) = data.split_at(1); + data = d; + let idx = idx[0]; + let var = enu + .variants() + .iter() + .find(|v| v.index() == idx) + .expect("variant"); + + let fields = var.fields(); + if fields.is_empty() { + if name == "Option" && *var.name() == "None" { + return ser.serialize_none(); + } + ser.serialize_unit_variant(name, var.index().into(), var.name()) + } else if is_tuple(fields) { + if fields.len() == 1 { + let ty = fields.first().unwrap().ty().type_info(); + let size = ty_data_size(&ty, data); + let val = Value::new(&data[..size], ty); + if name == "Option" && *var.name() == "Some" { + return ser.serialize_some(&val); + } + return ser.serialize_newtype_variant(name, idx.into(), var.name(), &val); + } + let mut s = + ser.serialize_tuple_variant(name, idx.into(), var.name(), fields.len())?; + for f in var.fields() { + let ty = f.ty().type_info(); + let size = ty_data_size(&ty, data); + let (val, reminder) = data.split_at(size); + data = reminder; + s.serialize_field(&Value::new(val, ty))?; + } + s.end() + } else { + let mut s = ser.serialize_struct_variant( + name, + var.index().into(), + var.name(), + fields.len(), + )?; + for f in var.fields() { + let ty = f.ty().type_info(); + let size = ty_data_size(&ty, data); + let (val, reminder) = data.split_at(size); + data = reminder; + s.serialize_field(f.name().unwrap(), &Value::new(val, ty))?; + } + s.end() + } + } + TypeDef::Sequence(seq) => { + let ty = seq.type_param().type_info(); + let (len, p_size) = sequence_len(data); + let mut data = &data[p_size..]; + let mut seq = ser.serialize_seq(Some(len))?; + + for _ in 0..len { + let ts = ty_data_size(&ty, data); + seq.serialize_element(&Value::new(&data[..ts], ty.clone()))?; + data = &data[ts..]; + } + seq.end() + } + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(tup) => { + let mut state = ser.serialize_tuple(tup.fields().len())?; + for f in tup.fields() { + let ty = f.type_info(); + let (val, reminder) = data.split_at(ty_data_size(&ty, data)); + data = reminder; + state.serialize_element(&Value::new(val, ty))?; + } + state.end() + } + TypeDef::Primitive(prm) => { + match prm { + Primitive::U8 => ser.serialize_u8(data[0]), + Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), + Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), + Primitive::U64 => ser.serialize_u64(LE::read_u64(data)), + Primitive::U128 => ser.serialize_u128(LE::read_u128(data)), + Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), + Primitive::I16 => ser.serialize_i16(LE::read_i16(data)), + Primitive::I32 => ser.serialize_i32(LE::read_i32(data)), + Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), + Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), + Primitive::Bool => ser.serialize_bool(data[0] != 0), + Primitive::Str => { + let (_, s) = sequence_len(data); + ser.serialize_str(str::from_utf8(&data[s..]).unwrap()) + } + // TODO: test if statements from here on actually work + Primitive::Char => { + let n = LE::read_u32(data); + ser.serialize_char(char::from_u32(n).unwrap()) + } + _ => ser.serialize_bytes(data), + } + } + TypeDef::Compact(_) => todo!(), + TypeDef::BitSequence(_) => todo!(), + } + } +} + +fn is_tuple(fields: &[Field]) -> bool { + fields.first().and_then(Field::name).is_none() +} + +fn ty_name(ty: &Type) -> &'static str { + ty.path().segments().last().copied().unwrap_or("") +} + +fn ty_data_size<'a>(ty: &Type, data: &'a [u8]) -> usize { + match ty.type_def() { + TypeDef::Primitive(p) => match p { + Primitive::U8 => mem::size_of::(), + Primitive::U16 => mem::size_of::(), + Primitive::U32 => mem::size_of::(), + Primitive::U64 => mem::size_of::(), + Primitive::U128 => mem::size_of::(), + Primitive::I8 => mem::size_of::(), + Primitive::I16 => mem::size_of::(), + Primitive::I32 => mem::size_of::(), + Primitive::I64 => mem::size_of::(), + Primitive::I128 => mem::size_of::(), + Primitive::Bool => mem::size_of::(), + Primitive::Char => mem::size_of::(), + Primitive::Str => { + let (l, p_size) = sequence_len(data); + l + p_size + } + _ => unimplemented!(), + }, + TypeDef::Composite(_) => todo!(), + TypeDef::Variant(e) => { + let var = e + .variants() + .iter() + .find(|v| v.index() == data[0]) + .expect("variant"); + + if var.fields().is_empty() { + 1 // unit variant + } else { + var.fields() + .iter() + .fold(1, |c, f| c + ty_data_size(&f.ty().type_info(), &data[c..])) + } + } + TypeDef::Sequence(s) => { + let (len, prefix_size) = sequence_len(data); + let ty = s.type_param().type_info(); + (0..len).fold(prefix_size, |c, _| c + ty_data_size(&ty, &data[c..])) + } + TypeDef::Array(_) => todo!(), + TypeDef::Tuple(_) => todo!(), + TypeDef::Compact(_) => todo!(), + TypeDef::BitSequence(_) => todo!(), + } +} + +fn sequence_len(data: &[u8]) -> (usize, usize) { + // need to peek at the data to know the length of sequence + // first byte(s) gives us a hint of the(compact encoded) length + // https://substrate.dev/docs/en/knowledgebase/advanced/codec#compactgeneral-integers + match data[0] % 0b100 { + 0 => ((data[0] >> 2).into(), 1), + 1 => (u16::from_le_bytes([(data[0] >> 2), data[1]]).into(), 2), + 2 => ( + u32::from_le_bytes([(data[0] >> 2), data[1], data[2], data[3]]) + .try_into() + .unwrap(), + 4, + ), + _ => todo!(), + } +} + +#[cfg(test)] +mod tests { + use std::error::Error; + + use super::*; + use parity_scale_codec::Encode; + use scale_info::TypeInfo; + use serde_json::to_value; + + #[test] + fn serialize_u8() -> Result<(), Box> { + let extract_value = u8::MAX; + let data = extract_value.encode(); + let info = u8::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_u16() -> Result<(), Box> { + let extract_value = u16::MAX; + let data = extract_value.encode(); + let info = u16::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_u32() -> Result<(), Box> { + let extract_value = u32::MAX; + let data = extract_value.encode(); + let info = u32::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_u64() -> Result<(), Box> { + let extract_value = u64::MAX; + let data = extract_value.encode(); + let info = u64::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_i16() -> Result<(), Box> { + let extract_value = i16::MAX; + let data = extract_value.encode(); + let info = i16::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_i32() -> Result<(), Box> { + let extract_value = i32::MAX; + let data = extract_value.encode(); + let info = i32::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_i64() -> Result<(), Box> { + let extract_value = i64::MAX; + let data = extract_value.encode(); + let info = i64::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_bool() -> Result<(), Box> { + let extract_value = true; + let data = extract_value.encode(); + let info = bool::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_u8array() -> Result<(), Box> { + let extract_value: Vec = [2u8, u8::MAX].into(); + let data = extract_value.encode(); + let info = Vec::::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_u16array() -> Result<(), Box> { + let extract_value: Vec = [2u16, u16::MAX].into(); + let data = extract_value.encode(); + let info = Vec::::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_u32array() -> Result<(), Box> { + let extract_value: Vec = [2u32, u32::MAX].into(); + let data = extract_value.encode(); + let info = Vec::::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_tuple() -> Result<(), Box> { + let extract_value: (i64, Vec, bool) = ( + i64::MIN, + vec!["hello".into(), "big".into(), "world".into()], + true, + ); + let data = extract_value.encode(); + let info = <(i64, Vec, bool)>::type_info(); + let val = Value::new(&data, info); + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_simple_u32struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: u32, + baz: u32, + } + let extract_value = Foo { + bar: 123, + baz: u32::MAX, + }; + let data = extract_value.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, info); + + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_simple_u8struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: u8, + baz: u8, + } + let extract_value = Foo { + bar: 123, + baz: u8::MAX, + }; + let data = extract_value.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, info); + + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_simple_u64struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: u64, + baz: u64, + } + let extract_value = Foo { + bar: 123, + baz: u64::MAX, + }; + let data = extract_value.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, info); + + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } + + #[test] + fn serialize_complex_struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + enum Bar { + This, + That(i16), + } + #[derive(Encode, Serialize, TypeInfo)] + struct Foo { + bar: Vec, + baz: Option, + } + let extract_value = Foo { + bar: [Bar::That(i16::MAX), Bar::This].into(), + baz: Some("aliquam malesuada bibendum arcu vitae".into()), + }; + let data = extract_value.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, info); + + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } +} From 840b8df0013e1d392844335c72066de588e92c12 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Sat, 4 Sep 2021 10:26:57 +0200 Subject: [PATCH 13/59] Serialize array and missing types --- src/value.rs | 157 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 113 insertions(+), 44 deletions(-) diff --git a/src/value.rs b/src/value.rs index 39781aa..ecc6142 100644 --- a/src/value.rs +++ b/src/value.rs @@ -8,9 +8,9 @@ use serde::Serialize; use std::convert::TryInto; use std::str; -/// A non-owning container for SCALE encoded data that can serialize types -/// directly with the help of a type registry and without using an intermediate -/// representation that requires allocating data. +/// A container for SCALE encoded data that can serialize types +/// directly with the help of a type registry and without using an +/// intermediate representation that requires allocating data. pub struct Value<'a> { data: &'a [u8], ty: Type, @@ -33,9 +33,18 @@ impl<'a> Serialize for Value<'a> { match self.ty.type_def() { TypeDef::Composite(cmp) => { let fields = cmp.fields(); + if fields.len() == 0 { + return ser.serialize_unit_struct(name); + } if is_tuple(fields) { - let state = ser.serialize_tuple_struct(name, fields.len())?; - // TODO!; + let mut state = ser.serialize_tuple_struct(name, fields.len())?; + for f in fields { + let ty = f.ty().type_info(); + let size = ty_data_size(&ty, data); + let (val, reminder) = data.split_at(size); + data = reminder; + state.serialize_field(&Value::new(val, ty))?; + } state.end() } else { let mut state = ser.serialize_struct(&name, fields.len())?; @@ -115,7 +124,16 @@ impl<'a> Serialize for Value<'a> { } seq.end() } - TypeDef::Array(_) => todo!(), + TypeDef::Array(arr) => { + let mut s = ser.serialize_tuple(arr.len().try_into().unwrap())?; + let ty = arr.type_param().type_info(); + for _ in 0..arr.len() { + let (val, reminder) = data.split_at(ty_data_size(&ty, data)); + data = reminder; + s.serialize_element(&Value::new(val, ty.clone()))?; + } + s.end() + } TypeDef::Tuple(tup) => { let mut state = ser.serialize_tuple(tup.fields().len())?; for f in tup.fields() { @@ -126,31 +144,29 @@ impl<'a> Serialize for Value<'a> { } state.end() } - TypeDef::Primitive(prm) => { - match prm { - Primitive::U8 => ser.serialize_u8(data[0]), - Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), - Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), - Primitive::U64 => ser.serialize_u64(LE::read_u64(data)), - Primitive::U128 => ser.serialize_u128(LE::read_u128(data)), - Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), - Primitive::I16 => ser.serialize_i16(LE::read_i16(data)), - Primitive::I32 => ser.serialize_i32(LE::read_i32(data)), - Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), - Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), - Primitive::Bool => ser.serialize_bool(data[0] != 0), - Primitive::Str => { - let (_, s) = sequence_len(data); - ser.serialize_str(str::from_utf8(&data[s..]).unwrap()) - } - // TODO: test if statements from here on actually work - Primitive::Char => { - let n = LE::read_u32(data); - ser.serialize_char(char::from_u32(n).unwrap()) - } - _ => ser.serialize_bytes(data), + TypeDef::Primitive(prm) => match prm { + Primitive::U8 => ser.serialize_u8(data[0]), + Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), + Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), + Primitive::U64 => ser.serialize_u64(LE::read_u64(data)), + Primitive::U128 => ser.serialize_u128(LE::read_u128(data)), + Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), + Primitive::I16 => ser.serialize_i16(LE::read_i16(data)), + Primitive::I32 => ser.serialize_i32(LE::read_i32(data)), + Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), + Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), + Primitive::Bool => ser.serialize_bool(data[0] != 0), + Primitive::Str => { + let (_, s) = sequence_len(data); + ser.serialize_str(str::from_utf8(&data[s..]).unwrap()) } - } + Primitive::Char => { + let n = LE::read_u32(data); + ser.serialize_char(char::from_u32(n).unwrap()) + } + Primitive::U256 => unimplemented!(), + Primitive::I256 => unimplemented!(), + }, TypeDef::Compact(_) => todo!(), TypeDef::BitSequence(_) => todo!(), } @@ -186,7 +202,10 @@ fn ty_data_size<'a>(ty: &Type, data: &'a [u8]) -> usize { } _ => unimplemented!(), }, - TypeDef::Composite(_) => todo!(), + TypeDef::Composite(c) => c + .fields() + .iter() + .fold(0, |c, f| c + ty_data_size(&f.ty().type_info(), &data[c..])), TypeDef::Variant(e) => { let var = e .variants() @@ -207,28 +226,39 @@ fn ty_data_size<'a>(ty: &Type, data: &'a [u8]) -> usize { let ty = s.type_param().type_info(); (0..len).fold(prefix_size, |c, _| c + ty_data_size(&ty, &data[c..])) } - TypeDef::Array(_) => todo!(), - TypeDef::Tuple(_) => todo!(), - TypeDef::Compact(_) => todo!(), + TypeDef::Array(a) => a.len().try_into().unwrap(), + TypeDef::Tuple(t) => t.fields().len(), + TypeDef::Compact(_) => compact_len(data), TypeDef::BitSequence(_) => todo!(), } } +fn compact_len(data: &[u8]) -> usize { + match data[0] % 0b100 { + 0 => 1, + 1 => 2, + 2 => 4, + _ => todo!(), + } +} + fn sequence_len(data: &[u8]) -> (usize, usize) { // need to peek at the data to know the length of sequence // first byte(s) gives us a hint of the(compact encoded) length // https://substrate.dev/docs/en/knowledgebase/advanced/codec#compactgeneral-integers - match data[0] % 0b100 { - 0 => ((data[0] >> 2).into(), 1), - 1 => (u16::from_le_bytes([(data[0] >> 2), data[1]]).into(), 2), - 2 => ( - u32::from_le_bytes([(data[0] >> 2), data[1], data[2], data[3]]) + let len = compact_len(data); + ( + match len { + 1 => (data[0] >> 2).into(), + 2 => u16::from_le_bytes([(data[0] >> 2), data[1]]).into(), + 4 => u32::from_le_bytes([(data[0] >> 2), data[1], data[2], data[3]]) .try_into() .unwrap(), - 4, - ), - _ => todo!(), - } + + _ => todo!(), + }, + len, + ) } #[cfg(test)] @@ -320,6 +350,17 @@ mod tests { Ok(()) } + // `char` not supported? + // #[test] + // fn serialize_char() -> Result<(), Box> { + // let extract_value = '⚖'; + // let data = extract_value.encode(); + // let info = char::type_info(); + // let val = Value::new(&data, info); + // assert_eq!(to_value(val)?, to_value(extract_value)?); + // Ok(()) + // } + #[test] fn serialize_u8array() -> Result<(), Box> { let extract_value: Vec = [2u8, u8::MAX].into(); @@ -422,7 +463,7 @@ mod tests { } #[test] - fn serialize_complex_struct() -> Result<(), Box> { + fn serialize_complex_struct_with_enum() -> Result<(), Box> { #[derive(Encode, Serialize, TypeInfo)] enum Bar { This, @@ -444,4 +485,32 @@ mod tests { assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } + + #[test] + fn serialize_tuple_struct() -> Result<(), Box> { + #[derive(Encode, Serialize, TypeInfo)] + struct Foo<'a>([u8; 4], (bool, Option<()>), Baz<'a>, Baz<'a>); + + #[derive(Encode, Serialize, TypeInfo)] + struct Bar; + + #[derive(Encode, Serialize, TypeInfo)] + enum Baz<'a> { + A(Bar), + B { bb: &'a str }, + } + + let extract_value = Foo( + [1, 2, 3, 4], + (false, None), + Baz::A(Bar), + Baz::B { bb: "lol" }, + ); + let data = extract_value.encode(); + let info = Foo::type_info(); + let val = Value::new(&data, info); + + assert_eq!(to_value(val)?, to_value(extract_value)?); + Ok(()) + } } From 6dff4cafd13962bdcde077f8f28cd50d4fd63dc4 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Sat, 4 Sep 2021 11:51:39 +0200 Subject: [PATCH 14/59] Refactor to simplify implementation --- src/value.rs | 126 ++++++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 61 deletions(-) diff --git a/src/value.rs b/src/value.rs index ecc6142..9e0aa28 100644 --- a/src/value.rs +++ b/src/value.rs @@ -5,6 +5,7 @@ use serde::ser::{ SerializeTupleVariant, }; use serde::Serialize; +use std::cell::Cell; use std::convert::TryInto; use std::str; @@ -14,11 +15,16 @@ use std::str; pub struct Value<'a> { data: &'a [u8], ty: Type, + idx: Cell, } impl<'a> Value<'a> { pub fn new(data: &'a [u8], ty: Type) -> Self { - Value { data, ty } + Value { + data, + ty, + idx: 0.into(), + } } } @@ -27,41 +33,32 @@ impl<'a> Serialize for Value<'a> { where S: serde::Serializer, { - let mut data = self.data; - let name = ty_name(&self.ty); + let name = self.ty_name(); match self.ty.type_def() { TypeDef::Composite(cmp) => { let fields = cmp.fields(); if fields.len() == 0 { - return ser.serialize_unit_struct(name); - } - if is_tuple(fields) { + ser.serialize_unit_struct(name) + } else if is_tuple(fields) { let mut state = ser.serialize_tuple_struct(name, fields.len())?; for f in fields { - let ty = f.ty().type_info(); - let size = ty_data_size(&ty, data); - let (val, reminder) = data.split_at(size); - data = reminder; - state.serialize_field(&Value::new(val, ty))?; + state.serialize_field(&self.sub_value(f.ty().type_info()))?; } state.end() } else { let mut state = ser.serialize_struct(&name, fields.len())?; for f in fields { - let ty = f.ty().type_info(); - let size = ty_data_size(&ty, data); - let (val, reminder) = data.split_at(size); - data = reminder; - state.serialize_field(f.name().unwrap(), &Value::new(val, ty))?; + state.serialize_field( + f.name().unwrap(), + &self.sub_value(f.ty().type_info()), + )?; } state.end() } } TypeDef::Variant(enu) => { - let (idx, d) = data.split_at(1); - data = d; - let idx = idx[0]; + let idx = self.extract_data(1)[0]; let var = enu .variants() .iter() @@ -77,8 +74,8 @@ impl<'a> Serialize for Value<'a> { } else if is_tuple(fields) { if fields.len() == 1 { let ty = fields.first().unwrap().ty().type_info(); - let size = ty_data_size(&ty, data); - let val = Value::new(&data[..size], ty); + let val = self.sub_value(ty); + if name == "Option" && *var.name() == "Some" { return ser.serialize_some(&val); } @@ -87,11 +84,7 @@ impl<'a> Serialize for Value<'a> { let mut s = ser.serialize_tuple_variant(name, idx.into(), var.name(), fields.len())?; for f in var.fields() { - let ty = f.ty().type_info(); - let size = ty_data_size(&ty, data); - let (val, reminder) = data.split_at(size); - data = reminder; - s.serialize_field(&Value::new(val, ty))?; + s.serialize_field(&self.sub_value(f.ty().type_info()))?; } s.end() } else { @@ -102,25 +95,19 @@ impl<'a> Serialize for Value<'a> { fields.len(), )?; for f in var.fields() { - let ty = f.ty().type_info(); - let size = ty_data_size(&ty, data); - let (val, reminder) = data.split_at(size); - data = reminder; - s.serialize_field(f.name().unwrap(), &Value::new(val, ty))?; + s.serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; } s.end() } } TypeDef::Sequence(seq) => { let ty = seq.type_param().type_info(); - let (len, p_size) = sequence_len(data); - let mut data = &data[p_size..]; + let (len, p_size) = sequence_len(self.remaining_data()); + self.advance_idx(p_size); let mut seq = ser.serialize_seq(Some(len))?; for _ in 0..len { - let ts = ty_data_size(&ty, data); - seq.serialize_element(&Value::new(&data[..ts], ty.clone()))?; - data = &data[ts..]; + seq.serialize_element(&self.sub_value(ty.clone()))?; } seq.end() } @@ -128,40 +115,36 @@ impl<'a> Serialize for Value<'a> { let mut s = ser.serialize_tuple(arr.len().try_into().unwrap())?; let ty = arr.type_param().type_info(); for _ in 0..arr.len() { - let (val, reminder) = data.split_at(ty_data_size(&ty, data)); - data = reminder; - s.serialize_element(&Value::new(val, ty.clone()))?; + s.serialize_element(&self.sub_value(ty.clone()))?; } s.end() } TypeDef::Tuple(tup) => { let mut state = ser.serialize_tuple(tup.fields().len())?; for f in tup.fields() { - let ty = f.type_info(); - let (val, reminder) = data.split_at(ty_data_size(&ty, data)); - data = reminder; - state.serialize_element(&Value::new(val, ty))?; + state.serialize_element(&self.sub_value(f.type_info()))?; } state.end() } TypeDef::Primitive(prm) => match prm { - Primitive::U8 => ser.serialize_u8(data[0]), - Primitive::U16 => ser.serialize_u16(LE::read_u16(data)), - Primitive::U32 => ser.serialize_u32(LE::read_u32(data)), - Primitive::U64 => ser.serialize_u64(LE::read_u64(data)), - Primitive::U128 => ser.serialize_u128(LE::read_u128(data)), - Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), - Primitive::I16 => ser.serialize_i16(LE::read_i16(data)), - Primitive::I32 => ser.serialize_i32(LE::read_i32(data)), - Primitive::I64 => ser.serialize_i64(LE::read_i64(data)), - Primitive::I128 => ser.serialize_i128(LE::read_i128(data)), - Primitive::Bool => ser.serialize_bool(data[0] != 0), + Primitive::U8 => ser.serialize_u8(self.remaining_data()[0]), + Primitive::U16 => ser.serialize_u16(LE::read_u16(self.remaining_data())), + Primitive::U32 => ser.serialize_u32(LE::read_u32(self.remaining_data())), + Primitive::U64 => ser.serialize_u64(LE::read_u64(self.remaining_data())), + Primitive::U128 => ser.serialize_u128(LE::read_u128(self.remaining_data())), + Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([self.remaining_data()[0]])), + Primitive::I16 => ser.serialize_i16(LE::read_i16(self.remaining_data())), + Primitive::I32 => ser.serialize_i32(LE::read_i32(self.remaining_data())), + Primitive::I64 => ser.serialize_i64(LE::read_i64(self.remaining_data())), + Primitive::I128 => ser.serialize_i128(LE::read_i128(self.remaining_data())), + Primitive::Bool => ser.serialize_bool(self.remaining_data()[0] != 0), Primitive::Str => { - let (_, s) = sequence_len(data); - ser.serialize_str(str::from_utf8(&data[s..]).unwrap()) + let (_, s) = sequence_len(self.remaining_data()); + self.advance_idx(s); + ser.serialize_str(str::from_utf8(self.remaining_data()).unwrap()) } Primitive::Char => { - let n = LE::read_u32(data); + let n = LE::read_u32(self.remaining_data()); ser.serialize_char(char::from_u32(n).unwrap()) } Primitive::U256 => unimplemented!(), @@ -173,12 +156,33 @@ impl<'a> Serialize for Value<'a> { } } -fn is_tuple(fields: &[Field]) -> bool { - fields.first().and_then(Field::name).is_none() +impl<'a> Value<'a> { + fn remaining_data(&self) -> &[u8] { + &self.data[self.idx.get()..] + } + + fn sub_value(&self, ty: Type) -> Value { + let size = ty_data_size(&ty, &self.remaining_data()); + Value::new(self.extract_data(size), ty) + } + + fn extract_data(&self, end: usize) -> &[u8] { + let (data, _) = &self.remaining_data().split_at(end); + self.advance_idx(end); + data + } + + fn advance_idx(&self, n: usize) { + self.idx.set(self.idx.get() + n) + } + + fn ty_name(&self) -> &'static str { + self.ty.path().segments().last().copied().unwrap_or("") + } } -fn ty_name(ty: &Type) -> &'static str { - ty.path().segments().last().copied().unwrap_or("") +fn is_tuple(fields: &[Field]) -> bool { + fields.first().and_then(Field::name).is_none() } fn ty_data_size<'a>(ty: &Type, data: &'a [u8]) -> usize { From 1150f54da4c5c46402b89923367a7050e637027d Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Tue, 7 Sep 2021 12:20:20 +0200 Subject: [PATCH 15/59] Make lib no_std --- .github/workflows/test.yml | 3 ++- Cargo.toml | 8 +++++- src/lib.rs | 1 + src/value.rs | 50 ++++++++++++++++++++------------------ 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 51e280c..d6a1cf5 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,6 +30,7 @@ jobs: - uses: actions-rs/cargo@v1 with: command: test + args: --features=std fmt: name: Rustfmt @@ -61,4 +62,4 @@ jobs: - uses: actions-rs/cargo@v1 with: command: clippy - args: -- -D warnings \ No newline at end of file + args: -- -D warnings diff --git a/Cargo.toml b/Cargo.toml index 3ce969f..e0fa5c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,15 +1,21 @@ [package] name = "scales" -version = "0.1.0" +description = "SCALE Serialization" +version = "0.2.0" authors = ["Daniel Olano "] edition = "2018" +repository = "https://github.com/valibre-org/scales" [dependencies] byteorder = { version = "1.4.3", default-features = false } scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } +[features] +std = ["scale-info/std"] + [dev-dependencies] +anyhow = "1.0.42" parity-scale-codec = { version = "2.2.0", features = ["derive"] } scale-info = { version = "0.10.0", features = ["derive"] } serde = { version = "1.0.126", features = ["derive"] } diff --git a/src/lib.rs b/src/lib.rs index 55ffc7f..33ee99b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,3 +1,4 @@ +#![cfg_attr(not(feature = "std"), no_std)] ///! # Scales ///! ///! Dynamic SCALE Serialization using `scale-info` type information. diff --git a/src/value.rs b/src/value.rs index 9e0aa28..525bc87 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,13 +1,13 @@ use byteorder::{ByteOrder, LE}; +use core::cell::Cell; +use core::convert::TryInto; +use core::str; use scale_info::{prelude::*, Field, Type, TypeDef, TypeDefPrimitive as Primitive}; use serde::ser::{ SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, SerializeTupleStruct, SerializeTupleVariant, }; use serde::Serialize; -use std::cell::Cell; -use std::convert::TryInto; -use std::str; /// A container for SCALE encoded data that can serialize types /// directly with the help of a type registry and without using an @@ -267,15 +267,17 @@ fn sequence_len(data: &[u8]) -> (usize, usize) { #[cfg(test)] mod tests { - use std::error::Error; - use super::*; + use anyhow::Error; use parity_scale_codec::Encode; - use scale_info::TypeInfo; + use scale_info::{ + prelude::{string::String, vec::Vec}, + TypeInfo, + }; use serde_json::to_value; #[test] - fn serialize_u8() -> Result<(), Box> { + fn serialize_u8() -> Result<(), Error> { let extract_value = u8::MAX; let data = extract_value.encode(); let info = u8::type_info(); @@ -285,7 +287,7 @@ mod tests { } #[test] - fn serialize_u16() -> Result<(), Box> { + fn serialize_u16() -> Result<(), Error> { let extract_value = u16::MAX; let data = extract_value.encode(); let info = u16::type_info(); @@ -295,7 +297,7 @@ mod tests { } #[test] - fn serialize_u32() -> Result<(), Box> { + fn serialize_u32() -> Result<(), Error> { let extract_value = u32::MAX; let data = extract_value.encode(); let info = u32::type_info(); @@ -305,7 +307,7 @@ mod tests { } #[test] - fn serialize_u64() -> Result<(), Box> { + fn serialize_u64() -> Result<(), Error> { let extract_value = u64::MAX; let data = extract_value.encode(); let info = u64::type_info(); @@ -315,7 +317,7 @@ mod tests { } #[test] - fn serialize_i16() -> Result<(), Box> { + fn serialize_i16() -> Result<(), Error> { let extract_value = i16::MAX; let data = extract_value.encode(); let info = i16::type_info(); @@ -325,7 +327,7 @@ mod tests { } #[test] - fn serialize_i32() -> Result<(), Box> { + fn serialize_i32() -> Result<(), Error> { let extract_value = i32::MAX; let data = extract_value.encode(); let info = i32::type_info(); @@ -335,7 +337,7 @@ mod tests { } #[test] - fn serialize_i64() -> Result<(), Box> { + fn serialize_i64() -> Result<(), Error> { let extract_value = i64::MAX; let data = extract_value.encode(); let info = i64::type_info(); @@ -345,7 +347,7 @@ mod tests { } #[test] - fn serialize_bool() -> Result<(), Box> { + fn serialize_bool() -> Result<(), Error> { let extract_value = true; let data = extract_value.encode(); let info = bool::type_info(); @@ -356,7 +358,7 @@ mod tests { // `char` not supported? // #[test] - // fn serialize_char() -> Result<(), Box> { + // fn serialize_char() -> Result<(), Error> { // let extract_value = '⚖'; // let data = extract_value.encode(); // let info = char::type_info(); @@ -366,7 +368,7 @@ mod tests { // } #[test] - fn serialize_u8array() -> Result<(), Box> { + fn serialize_u8array() -> Result<(), Error> { let extract_value: Vec = [2u8, u8::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); @@ -376,7 +378,7 @@ mod tests { } #[test] - fn serialize_u16array() -> Result<(), Box> { + fn serialize_u16array() -> Result<(), Error> { let extract_value: Vec = [2u16, u16::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); @@ -386,7 +388,7 @@ mod tests { } #[test] - fn serialize_u32array() -> Result<(), Box> { + fn serialize_u32array() -> Result<(), Error> { let extract_value: Vec = [2u32, u32::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); @@ -396,7 +398,7 @@ mod tests { } #[test] - fn serialize_tuple() -> Result<(), Box> { + fn serialize_tuple() -> Result<(), Error> { let extract_value: (i64, Vec, bool) = ( i64::MIN, vec!["hello".into(), "big".into(), "world".into()], @@ -410,7 +412,7 @@ mod tests { } #[test] - fn serialize_simple_u32struct() -> Result<(), Box> { + fn serialize_simple_u32struct() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] struct Foo { bar: u32, @@ -429,7 +431,7 @@ mod tests { } #[test] - fn serialize_simple_u8struct() -> Result<(), Box> { + fn serialize_simple_u8struct() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] struct Foo { bar: u8, @@ -448,7 +450,7 @@ mod tests { } #[test] - fn serialize_simple_u64struct() -> Result<(), Box> { + fn serialize_simple_u64struct() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] struct Foo { bar: u64, @@ -467,7 +469,7 @@ mod tests { } #[test] - fn serialize_complex_struct_with_enum() -> Result<(), Box> { + fn serialize_complex_struct_with_enum() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] enum Bar { This, @@ -491,7 +493,7 @@ mod tests { } #[test] - fn serialize_tuple_struct() -> Result<(), Box> { + fn serialize_tuple_struct() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] struct Foo<'a>([u8; 4], (bool, Option<()>), Baz<'a>, Baz<'a>); From f3b6c979a05915ddb2bbede183fc35de763c15a6 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 9 Sep 2021 11:06:01 +0200 Subject: [PATCH 16/59] Update README.md --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 24b7969..1347503 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ -# scales -SCALE (De)Serialization +# scales - SCALE Serialization + +Making use of [type information](https://github.com/paritytech/scale-info) this library allows for conversion of SCALE encoded data(wrapped in a `scales::Value`) to any format that implements `Serialize` including dynamic types like `serde_json::Value` for example. (🚧 WIP 🚧)The opposite conversion of arbitrary data(e.g. JSON) to SCALE binary format is also possible with the `serializer` feature. From ab945867081f9edfbddfd1f8778e045cd63e1532 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 19 Aug 2021 00:29:47 +0200 Subject: [PATCH 17/59] Supoprt SCALE to Any format via serde serializer --- src/lib.rs | 2 + src/serializer.rs | 317 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 319 insertions(+) create mode 100644 src/serializer.rs diff --git a/src/lib.rs b/src/lib.rs index 33ee99b..11aec34 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,8 @@ ///! ///! Dynamic SCALE Serialization using `scale-info` type information. ///! +mod serializer; mod value; +pub use serializer::{to_bytes, Serializer}; pub use value::Value; diff --git a/src/serializer.rs b/src/serializer.rs new file mode 100644 index 0000000..c4d835f --- /dev/null +++ b/src/serializer.rs @@ -0,0 +1,317 @@ +use error::Error; +use serde::{ser, Serialize}; + +pub fn to_bytes(value: &T) -> Result, ()> +where + T: Serialize, +{ + let mut serializer = Serializer {}; + value.serialize(&mut serializer)?; + Ok(vec![]) +} + +pub struct Serializer {} + +impl<'a> ser::Serializer for &'a mut Serializer { + type Ok = (); + type Error = Error; + + type SerializeSeq = Self; + type SerializeTuple = Self; + type SerializeTupleStruct = Self; + type SerializeTupleVariant = Self; + type SerializeMap = Self; + type SerializeStruct = Self; + type SerializeStructVariant = Self; + + fn serialize_bool(self, v: bool) -> Result { + todo!() + } + + fn serialize_i8(self, v: i8) -> Result { + todo!() + } + + fn serialize_i16(self, v: i16) -> Result { + todo!() + } + + fn serialize_i32(self, v: i32) -> Result { + todo!() + } + + fn serialize_i64(self, v: i64) -> Result { + todo!() + } + + fn serialize_u8(self, v: u8) -> Result { + todo!() + } + + fn serialize_u16(self, v: u16) -> Result { + todo!() + } + + fn serialize_u32(self, v: u32) -> Result { + todo!() + } + + fn serialize_u64(self, v: u64) -> Result { + todo!() + } + + fn serialize_f32(self, v: f32) -> Result { + todo!() + } + + fn serialize_f64(self, v: f64) -> Result { + todo!() + } + + fn serialize_char(self, v: char) -> Result { + todo!() + } + + fn serialize_str(self, v: &str) -> Result { + todo!() + } + + fn serialize_bytes(self, v: &[u8]) -> Result { + todo!() + } + + fn serialize_none(self) -> Result { + todo!() + } + + fn serialize_some(self, value: &T) -> Result + where + T: Serialize, + { + todo!() + } + + fn serialize_unit(self) -> Result { + todo!() + } + + fn serialize_unit_struct(self, name: &'static str) -> Result { + todo!() + } + + fn serialize_unit_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + ) -> Result { + todo!() + } + + fn serialize_newtype_struct( + self, + name: &'static str, + value: &T, + ) -> Result + where + T: Serialize, + { + todo!() + } + + fn serialize_newtype_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result + where + T: Serialize, + { + todo!() + } + + fn serialize_seq(self, len: Option) -> Result { + todo!() + } + + fn serialize_tuple(self, len: usize) -> Result { + todo!() + } + + fn serialize_tuple_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + todo!() + } + + fn serialize_tuple_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + todo!() + } + + fn serialize_map(self, len: Option) -> Result { + todo!() + } + + fn serialize_struct( + self, + name: &'static str, + len: usize, + ) -> Result { + todo!() + } + + fn serialize_struct_variant( + self, + name: &'static str, + variant_index: u32, + variant: &'static str, + len: usize, + ) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeMap for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeSeq for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeStruct for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeStructVariant for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeTuple for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeTupleStruct for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +impl<'a> ser::SerializeTupleVariant for &'a mut Serializer { + type Ok = (); + type Error = Error; + + fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + where + T: Serialize, + { + todo!() + } + + fn end(self) -> Result { + todo!() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_struct() {} +} From f42bf8081028ef38d7eed12ee6bce7ac3356902c Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 8 Sep 2021 09:00:32 +0200 Subject: [PATCH 18/59] Setup serializer no_std support --- Cargo.toml | 11 ++-- src/lib.rs | 5 +- src/serializer.rs | 132 ++++++++++++++++++++++++---------------------- 3 files changed, 81 insertions(+), 67 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e0fa5c8..c87833e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,15 +8,18 @@ repository = "https://github.com/valibre-org/scales" [dependencies] byteorder = { version = "1.4.3", default-features = false } +core2 = { version = "0.3.1", default-features = false, optional = true } scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } [features] -std = ["scale-info/std"] +#default = ["std", "serializer"] +std = ["scale-info/std", "core2/std"] +serializer = ["core2"] [dev-dependencies] anyhow = "1.0.42" parity-scale-codec = { version = "2.2.0", features = ["derive"] } -scale-info = { version = "0.10.0", features = ["derive"] } -serde = { version = "1.0.126", features = ["derive"] } -serde_json = "1.0.66" +scale-info = { version = "0.10.0", default-features = false, features = ["derive"] } +serde = { version = "1.0.126", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.66", default-features = false, features = ["alloc"] } diff --git a/src/lib.rs b/src/lib.rs index 11aec34..fccfbc2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,8 +3,11 @@ ///! ///! Dynamic SCALE Serialization using `scale-info` type information. ///! + +#[cfg(feature = "serializer")] mod serializer; mod value; -pub use serializer::{to_bytes, Serializer}; +#[cfg(feature = "serializer")] +pub use serializer::{to_writer, Serializer}; pub use value::Value; diff --git a/src/serializer.rs b/src/serializer.rs index c4d835f..95ec8b5 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,15 +1,19 @@ -use error::Error; +use core2::io; use serde::{ser, Serialize}; -pub fn to_bytes(value: &T) -> Result, ()> +type Result = core::result::Result; + +pub fn to_writer(writer: W, value: &T) -> Result<()> where T: Serialize, + W: io::Write, { let mut serializer = Serializer {}; value.serialize(&mut serializer)?; - Ok(vec![]) + Ok(()) } +/// pub struct Serializer {} impl<'a> ser::Serializer for &'a mut Serializer { @@ -24,78 +28,78 @@ impl<'a> ser::Serializer for &'a mut Serializer { type SerializeStruct = Self; type SerializeStructVariant = Self; - fn serialize_bool(self, v: bool) -> Result { + fn serialize_bool(self, v: bool) -> Result { todo!() } - fn serialize_i8(self, v: i8) -> Result { + fn serialize_i8(self, v: i8) -> Result { todo!() } - fn serialize_i16(self, v: i16) -> Result { + fn serialize_i16(self, v: i16) -> Result { todo!() } - fn serialize_i32(self, v: i32) -> Result { + fn serialize_i32(self, v: i32) -> Result { todo!() } - fn serialize_i64(self, v: i64) -> Result { + fn serialize_i64(self, v: i64) -> Result { todo!() } - fn serialize_u8(self, v: u8) -> Result { + fn serialize_u8(self, v: u8) -> Result { todo!() } - fn serialize_u16(self, v: u16) -> Result { + fn serialize_u16(self, v: u16) -> Result { todo!() } - fn serialize_u32(self, v: u32) -> Result { + fn serialize_u32(self, v: u32) -> Result { todo!() } - fn serialize_u64(self, v: u64) -> Result { + fn serialize_u64(self, v: u64) -> Result { todo!() } - fn serialize_f32(self, v: f32) -> Result { + fn serialize_f32(self, v: f32) -> Result { todo!() } - fn serialize_f64(self, v: f64) -> Result { + fn serialize_f64(self, v: f64) -> Result { todo!() } - fn serialize_char(self, v: char) -> Result { + fn serialize_char(self, v: char) -> Result { todo!() } - fn serialize_str(self, v: &str) -> Result { + fn serialize_str(self, v: &str) -> Result { todo!() } - fn serialize_bytes(self, v: &[u8]) -> Result { + fn serialize_bytes(self, v: &[u8]) -> Result { todo!() } - fn serialize_none(self) -> Result { + fn serialize_none(self) -> Result { todo!() } - fn serialize_some(self, value: &T) -> Result + fn serialize_some(self, value: &T) -> Result where T: Serialize, { todo!() } - fn serialize_unit(self) -> Result { + fn serialize_unit(self) -> Result { todo!() } - fn serialize_unit_struct(self, name: &'static str) -> Result { + fn serialize_unit_struct(self, name: &'static str) -> Result { todo!() } @@ -104,15 +108,11 @@ impl<'a> ser::Serializer for &'a mut Serializer { name: &'static str, variant_index: u32, variant: &'static str, - ) -> Result { + ) -> Result { todo!() } - fn serialize_newtype_struct( - self, - name: &'static str, - value: &T, - ) -> Result + fn serialize_newtype_struct(self, name: &'static str, value: &T) -> Result where T: Serialize, { @@ -125,18 +125,18 @@ impl<'a> ser::Serializer for &'a mut Serializer { variant_index: u32, variant: &'static str, value: &T, - ) -> Result + ) -> Result where T: Serialize, { todo!() } - fn serialize_seq(self, len: Option) -> Result { + fn serialize_seq(self, len: Option) -> Result { todo!() } - fn serialize_tuple(self, len: usize) -> Result { + fn serialize_tuple(self, len: usize) -> Result { todo!() } @@ -144,7 +144,7 @@ impl<'a> ser::Serializer for &'a mut Serializer { self, name: &'static str, len: usize, - ) -> Result { + ) -> Result { todo!() } @@ -154,19 +154,15 @@ impl<'a> ser::Serializer for &'a mut Serializer { variant_index: u32, variant: &'static str, len: usize, - ) -> Result { + ) -> Result { todo!() } - fn serialize_map(self, len: Option) -> Result { + fn serialize_map(self, len: Option) -> Result { todo!() } - fn serialize_struct( - self, - name: &'static str, - len: usize, - ) -> Result { + fn serialize_struct(self, name: &'static str, len: usize) -> Result { todo!() } @@ -176,7 +172,7 @@ impl<'a> ser::Serializer for &'a mut Serializer { variant_index: u32, variant: &'static str, len: usize, - ) -> Result { + ) -> Result { todo!() } } @@ -185,21 +181,21 @@ impl<'a> ser::SerializeMap for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_key(&mut self, key: &T) -> Result<(), Self::Error> + fn serialize_key(&mut self, key: &T) -> Result<()> where T: Serialize, { todo!() } - fn serialize_value(&mut self, value: &T) -> Result<(), Self::Error> + fn serialize_value(&mut self, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { todo!() } } @@ -208,14 +204,14 @@ impl<'a> ser::SerializeSeq for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + fn serialize_element(&mut self, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { todo!() } } @@ -224,18 +220,14 @@ impl<'a> ser::SerializeStruct for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field( - &mut self, - key: &'static str, - value: &T, - ) -> Result<(), Self::Error> + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { todo!() } } @@ -244,18 +236,14 @@ impl<'a> ser::SerializeStructVariant for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field( - &mut self, - key: &'static str, - value: &T, - ) -> Result<(), Self::Error> + fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { todo!() } } @@ -264,14 +252,14 @@ impl<'a> ser::SerializeTuple for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_element(&mut self, value: &T) -> Result<(), Self::Error> + fn serialize_element(&mut self, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { todo!() } } @@ -280,14 +268,14 @@ impl<'a> ser::SerializeTupleStruct for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + fn serialize_field(&mut self, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { todo!() } } @@ -296,14 +284,34 @@ impl<'a> ser::SerializeTupleVariant for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<(), Self::Error> + fn serialize_field(&mut self, value: &T) -> Result<()> where T: Serialize, { todo!() } - fn end(self) -> Result { + fn end(self) -> Result { + todo!() + } +} + +#[derive(Debug)] +pub enum Error {} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + todo!() + } +} + +impl ser::StdError for Error {} + +impl ser::Error for Error { + fn custom(_msg: T) -> Self + where + T: core::fmt::Display, + { todo!() } } From cb0c4f58c22613088753a3194a73eb299d4b71e8 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 9 Sep 2021 10:49:08 +0200 Subject: [PATCH 19/59] Serialize primitives --- Cargo.toml | 7 +- src/serializer.rs | 325 ++++++++++++++++++++++++++++++++++++---------- src/value.rs | 2 +- 3 files changed, 264 insertions(+), 70 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c87833e..07efdd0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,18 +8,19 @@ repository = "https://github.com/valibre-org/scales" [dependencies] byteorder = { version = "1.4.3", default-features = false } +codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, optional = true } core2 = { version = "0.3.1", default-features = false, optional = true } scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } [features] -#default = ["std", "serializer"] +default = ["std", "serializer"] std = ["scale-info/std", "core2/std"] -serializer = ["core2"] +serializer = ["core2", "codec"] [dev-dependencies] anyhow = "1.0.42" -parity-scale-codec = { version = "2.2.0", features = ["derive"] } +codec = { version = "2.2.0", package = "parity-scale-codec", features = ["derive"] } scale-info = { version = "0.10.0", default-features = false, features = ["derive"] } serde = { version = "1.0.126", default-features = false, features = ["derive"] } serde_json = { version = "1.0.66", default-features = false, features = ["alloc"] } diff --git a/src/serializer.rs b/src/serializer.rs index 95ec8b5..0c3f594 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,3 +1,5 @@ +use codec::{Compact, Encode}; +use core::fmt; use core2::io; use serde::{ser, Serialize}; @@ -8,15 +10,21 @@ where T: Serialize, W: io::Write, { - let mut serializer = Serializer {}; + let mut serializer = Serializer::new(writer); value.serialize(&mut serializer)?; Ok(()) } /// -pub struct Serializer {} +pub struct Serializer(W); -impl<'a> ser::Serializer for &'a mut Serializer { +impl Serializer { + pub fn new(writer: W) -> Self { + Serializer(writer) + } +} + +impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { type Ok = (); type Error = Error; @@ -29,90 +37,97 @@ impl<'a> ser::Serializer for &'a mut Serializer { type SerializeStructVariant = Self; fn serialize_bool(self, v: bool) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_i8(self, v: i8) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_i16(self, v: i16) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_i32(self, v: i32) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_i64(self, v: i64) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_u8(self, v: u8) -> Result { - todo!() + self.0.write(&[v])?; + Ok(()) } fn serialize_u16(self, v: u16) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_u32(self, v: u32) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_u64(self, v: u64) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } - fn serialize_f32(self, v: f32) -> Result { - todo!() + fn serialize_f32(self, _v: f32) -> Result { + unimplemented!() } - fn serialize_f64(self, v: f64) -> Result { - todo!() + fn serialize_f64(self, _v: f64) -> Result { + unimplemented!() } - fn serialize_char(self, v: char) -> Result { + fn serialize_char(self, _v: char) -> Result { todo!() } fn serialize_str(self, v: &str) -> Result { - todo!() + Ok(v.encode_to(&mut self.0)) } fn serialize_bytes(self, v: &[u8]) -> Result { - todo!() + self.0.write_all(v).map_err(Error::from) } fn serialize_none(self) -> Result { - todo!() + self.0.write(&[0x00])?; + Ok(()) } fn serialize_some(self, value: &T) -> Result where T: Serialize, { - todo!() + self.0.write(&[0x01])?; + value.serialize(self) } fn serialize_unit(self) -> Result { todo!() } - fn serialize_unit_struct(self, name: &'static str) -> Result { + fn serialize_unit_struct(self, _name: &'static str) -> Result { todo!() } fn serialize_unit_variant( self, - name: &'static str, + __name: &'static str, variant_index: u32, - variant: &'static str, + _variant: &'static str, ) -> Result { - todo!() + (variant_index as u8).serialize(self) } - fn serialize_newtype_struct(self, name: &'static str, value: &T) -> Result + fn serialize_newtype_struct( + self, + _name: &'static str, + _value: &T, + ) -> Result where T: Serialize, { @@ -121,10 +136,10 @@ impl<'a> ser::Serializer for &'a mut Serializer { fn serialize_newtype_variant( self, - name: &'static str, - variant_index: u32, - variant: &'static str, - value: &T, + __name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, ) -> Result where T: Serialize, @@ -133,62 +148,63 @@ impl<'a> ser::Serializer for &'a mut Serializer { } fn serialize_seq(self, len: Option) -> Result { - todo!() + Compact(len.expect("known length") as u64).encode_to(&mut self.0); + Ok(self) } - fn serialize_tuple(self, len: usize) -> Result { - todo!() + fn serialize_tuple(self, _len: usize) -> Result { + Ok(self) } fn serialize_tuple_struct( self, - name: &'static str, - len: usize, + __name: &'static str, + __len: usize, ) -> Result { todo!() } fn serialize_tuple_variant( self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, + __name: &'static str, + _variant_index: u32, + _variant: &'static str, + __len: usize, ) -> Result { todo!() } - fn serialize_map(self, len: Option) -> Result { + fn serialize_map(self, _len: Option) -> Result { todo!() } - fn serialize_struct(self, name: &'static str, len: usize) -> Result { + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { todo!() } fn serialize_struct_variant( self, - name: &'static str, - variant_index: u32, - variant: &'static str, - len: usize, + __name: &'static str, + _variant_index: u32, + _variant: &'static str, + __len: usize, ) -> Result { todo!() } } -impl<'a> ser::SerializeMap for &'a mut Serializer { +impl<'a, W> ser::SerializeMap for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_key(&mut self, key: &T) -> Result<()> + fn serialize_key(&mut self, _key: &T) -> Result<()> where T: Serialize, { todo!() } - fn serialize_value(&mut self, value: &T) -> Result<()> + fn serialize_value(&mut self, _value: &T) -> Result<()> where T: Serialize, { @@ -200,7 +216,10 @@ impl<'a> ser::SerializeMap for &'a mut Serializer { } } -impl<'a> ser::SerializeSeq for &'a mut Serializer { +impl<'a, W> ser::SerializeSeq for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; @@ -208,19 +227,19 @@ impl<'a> ser::SerializeSeq for &'a mut Serializer { where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } -impl<'a> ser::SerializeStruct for &'a mut Serializer { +impl<'a, W> ser::SerializeStruct for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + fn serialize_field(&mut self, _key: &'static str, _value: &T) -> Result<()> where T: Serialize, { @@ -232,11 +251,11 @@ impl<'a> ser::SerializeStruct for &'a mut Serializer { } } -impl<'a> ser::SerializeStructVariant for &'a mut Serializer { +impl<'a, W> ser::SerializeStructVariant for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field(&mut self, key: &'static str, value: &T) -> Result<()> + fn serialize_field(&mut self, _key: &'static str, _value: &T) -> Result<()> where T: Serialize, { @@ -248,7 +267,10 @@ impl<'a> ser::SerializeStructVariant for &'a mut Serializer { } } -impl<'a> ser::SerializeTuple for &'a mut Serializer { +impl<'a, W> ser::SerializeTuple for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; @@ -256,19 +278,19 @@ impl<'a> ser::SerializeTuple for &'a mut Serializer { where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } -impl<'a> ser::SerializeTupleStruct for &'a mut Serializer { +impl<'a, W> ser::SerializeTupleStruct for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<()> + fn serialize_field(&mut self, _value: &T) -> Result<()> where T: Serialize, { @@ -280,11 +302,11 @@ impl<'a> ser::SerializeTupleStruct for &'a mut Serializer { } } -impl<'a> ser::SerializeTupleVariant for &'a mut Serializer { +impl<'a, W> ser::SerializeTupleVariant for &'a mut Serializer { type Ok = (); type Error = Error; - fn serialize_field(&mut self, value: &T) -> Result<()> + fn serialize_field(&mut self, _value: &T) -> Result<()> where T: Serialize, { @@ -299,8 +321,8 @@ impl<'a> ser::SerializeTupleVariant for &'a mut Serializer { #[derive(Debug)] pub enum Error {} -impl core::fmt::Display for Error { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Display for Error { + fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { todo!() } } @@ -310,16 +332,187 @@ impl ser::StdError for Error {} impl ser::Error for Error { fn custom(_msg: T) -> Self where - T: core::fmt::Display, + T: fmt::Display, { todo!() } } +impl From for Error { + fn from(_: io::Error) -> Self { + todo!() + } +} + #[cfg(test)] mod tests { use super::*; + use core::mem::size_of; #[test] - fn test_struct() {} + fn test_primitive_u8() -> Result<()> { + let mut out = [0u8]; + to_writer(&mut out[..], &123u8)?; + + let expected = [123]; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_primitive_u16() -> Result<()> { + const INPUT: u16 = 0xFF_EE; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_ref(), expected); + Ok(()) + } + + #[test] + fn test_primitive_u32() -> Result<()> { + const INPUT: u32 = 0xFF_EE_DD_CC; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_ref(), expected); + Ok(()) + } + + #[test] + fn test_primitive_u64() -> Result<()> { + const INPUT: u64 = 0xFF_EE_DD_CC__BB_AA_99_88; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + + #[test] + fn test_primitive_i16() -> Result<()> { + const INPUT: i16 = i16::MIN; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + + #[test] + fn test_primitive_i32() -> Result<()> { + const INPUT: i32 = i32::MIN; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + + #[test] + fn test_primitive_i64() -> Result<()> { + const INPUT: i64 = i64::MIN; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + + #[test] + fn test_primitive_bool() -> Result<()> { + const INPUT: bool = true; + let mut out = [0u8]; + let expected = INPUT.encode(); + + to_writer(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + + #[test] + fn test_str() -> Result<()> { + const INPUT: &str = "ac orci phasellus egestas tellus rutrum tellus pellentesque"; + let mut out = Vec::::new(); + let expected = INPUT.encode(); + + to_writer(&mut out, &INPUT)?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_bytes() -> Result<()> { + const INPUT: &[u8] = b"dictumst quisque sagittis purus sit amet volutpat consequat"; + let mut out = Vec::::new(); + let expected = INPUT.encode(); + + to_writer(&mut out, &INPUT)?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_tuple_simple() -> Result<()> { + const INPUT: (u8, bool, u64) = (0xD0, false, u64::MAX); + let mut out = Vec::::new(); + let expected = INPUT.encode(); + + to_writer(&mut out, &INPUT)?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_enum_simple() -> Result<()> { + #[derive(Serialize, Encode)] + enum X { + _A, + B, + } + + const INPUT: X = X::B; + let mut out = Vec::::new(); + let expected = INPUT.encode(); + + to_writer(&mut out, &INPUT)?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_tuple_enum_mix() -> Result<()> { + #[derive(Serialize, Encode)] + enum X { + A, + B, + } + + let input: (Option<()>, Option, X, X) = (None, Some("hello".into()), X::A, X::B); + let mut out = Vec::::new(); + let expected = input.encode(); + + to_writer(&mut out, &input)?; + + assert_eq!(out, expected); + Ok(()) + } } diff --git a/src/value.rs b/src/value.rs index 525bc87..a897c23 100644 --- a/src/value.rs +++ b/src/value.rs @@ -269,7 +269,7 @@ fn sequence_len(data: &[u8]) -> (usize, usize) { mod tests { use super::*; use anyhow::Error; - use parity_scale_codec::Encode; + use codec::Encode; use scale_info::{ prelude::{string::String, vec::Vec}, TypeInfo, From d88dd61f974e12ba3b969392387fb08c52d65481 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 9 Sep 2021 15:41:28 +0200 Subject: [PATCH 20/59] Serializer serialize complex types --- Cargo.toml | 2 +- README.md | 2 +- src/serializer.rs | 211 ++++++++++++++++++++++++++++++++++------------ 3 files changed, 157 insertions(+), 58 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 07efdd0..68e505e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scales" description = "SCALE Serialization" -version = "0.2.0" +version = "0.3.0" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/valibre-org/scales" diff --git a/README.md b/README.md index 1347503..83ba598 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # scales - SCALE Serialization -Making use of [type information](https://github.com/paritytech/scale-info) this library allows for conversion of SCALE encoded data(wrapped in a `scales::Value`) to any format that implements `Serialize` including dynamic types like `serde_json::Value` for example. (🚧 WIP 🚧)The opposite conversion of arbitrary data(e.g. JSON) to SCALE binary format is also possible with the `serializer` feature. +Making use of [type information](https://github.com/paritytech/scale-info) this library allows for conversion of SCALE encoded data(wrapped in a `scales::Value`) to any format that implements `Serialize` including dynamic types like `serde_json::Value` for example. The opposite conversion of arbitrary data(e.g. JSON) to SCALE binary format is also possible with the `serializer` feature. diff --git a/src/serializer.rs b/src/serializer.rs index 0c3f594..90083fb 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -15,7 +15,6 @@ where Ok(()) } -/// pub struct Serializer(W); impl Serializer { @@ -24,7 +23,10 @@ impl Serializer { } } -impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { +impl<'a, W> ser::Serializer for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; @@ -57,8 +59,7 @@ impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { } fn serialize_u8(self, v: u8) -> Result { - self.0.write(&[v])?; - Ok(()) + self.0.write_all(&[v]).map_err(Error::from) } fn serialize_u16(self, v: u16) -> Result { @@ -82,7 +83,7 @@ impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { } fn serialize_char(self, _v: char) -> Result { - todo!() + unimplemented!() } fn serialize_str(self, v: &str) -> Result { @@ -94,24 +95,23 @@ impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { } fn serialize_none(self) -> Result { - self.0.write(&[0x00])?; - Ok(()) + self.0.write_all(&[0x00]).map_err(Error::from) } fn serialize_some(self, value: &T) -> Result where T: Serialize, { - self.0.write(&[0x01])?; + self.0.write_all(&[0x01])?; value.serialize(self) } fn serialize_unit(self) -> Result { - todo!() + Ok(()) } fn serialize_unit_struct(self, _name: &'static str) -> Result { - todo!() + Ok(()) } fn serialize_unit_variant( @@ -123,28 +123,25 @@ impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { (variant_index as u8).serialize(self) } - fn serialize_newtype_struct( - self, - _name: &'static str, - _value: &T, - ) -> Result + fn serialize_newtype_struct(self, _name: &'static str, value: &T) -> Result where T: Serialize, { - todo!() + value.serialize(self) } fn serialize_newtype_variant( self, __name: &'static str, - _variant_index: u32, + variant_index: u32, _variant: &'static str, - _value: &T, + value: &T, ) -> Result where T: Serialize, { - todo!() + self.0.write_all(&[variant_index as u8])?; + value.serialize(self) } fn serialize_seq(self, len: Option) -> Result { @@ -161,58 +158,64 @@ impl<'a, W: io::Write> ser::Serializer for &'a mut Serializer { __name: &'static str, __len: usize, ) -> Result { - todo!() + Ok(self) } fn serialize_tuple_variant( self, __name: &'static str, - _variant_index: u32, + variant_index: u32, _variant: &'static str, __len: usize, ) -> Result { - todo!() + self.0.write_all(&[variant_index as u8])?; + Ok(self) } - fn serialize_map(self, _len: Option) -> Result { - todo!() + fn serialize_map(self, len: Option) -> Result { + Compact(len.expect("known length") as u64).encode_to(&mut self.0); + Ok(self) } fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { - todo!() + Ok(self) } fn serialize_struct_variant( self, __name: &'static str, - _variant_index: u32, + variant_index: u32, _variant: &'static str, __len: usize, ) -> Result { - todo!() + self.0.write_all(&[variant_index as u8])?; + Ok(self) } } -impl<'a, W> ser::SerializeMap for &'a mut Serializer { +impl<'a, W> ser::SerializeMap for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; - fn serialize_key(&mut self, _key: &T) -> Result<()> + fn serialize_key(&mut self, key: &T) -> Result<()> where T: Serialize, { - todo!() + key.serialize(&mut **self) } - fn serialize_value(&mut self, _value: &T) -> Result<()> + fn serialize_value(&mut self, value: &T) -> Result<()> where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } @@ -235,35 +238,41 @@ where } } -impl<'a, W> ser::SerializeStruct for &'a mut Serializer { +impl<'a, W> ser::SerializeStruct for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; - fn serialize_field(&mut self, _key: &'static str, _value: &T) -> Result<()> + fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } -impl<'a, W> ser::SerializeStructVariant for &'a mut Serializer { +impl<'a, W> ser::SerializeStructVariant for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; - fn serialize_field(&mut self, _key: &'static str, _value: &T) -> Result<()> + fn serialize_field(&mut self, _key: &'static str, value: &T) -> Result<()> where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } @@ -286,61 +295,73 @@ where } } -impl<'a, W> ser::SerializeTupleStruct for &'a mut Serializer { +impl<'a, W> ser::SerializeTupleStruct for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; - fn serialize_field(&mut self, _value: &T) -> Result<()> + fn serialize_field(&mut self, value: &T) -> Result<()> where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } -impl<'a, W> ser::SerializeTupleVariant for &'a mut Serializer { +impl<'a, W> ser::SerializeTupleVariant for &'a mut Serializer +where + W: io::Write, +{ type Ok = (); type Error = Error; - fn serialize_field(&mut self, _value: &T) -> Result<()> + fn serialize_field(&mut self, value: &T) -> Result<()> where T: Serialize, { - todo!() + value.serialize(&mut **self) } fn end(self) -> Result { - todo!() + Ok(()) } } #[derive(Debug)] -pub enum Error {} +pub enum Error { + Ser(String), + Io(io::Error), +} impl fmt::Display for Error { - fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { - todo!() + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Ser(msg) => write!(f, "{}", msg), + Error::Io(e) => write!(f, "{}", e), + } } } impl ser::StdError for Error {} impl ser::Error for Error { - fn custom(_msg: T) -> Self + fn custom(msg: T) -> Self where T: fmt::Display, { - todo!() + Error::Ser(msg.to_string()) } } impl From for Error { - fn from(_: io::Error) -> Self { - todo!() + fn from(e: io::Error) -> Self { + Error::Io(e) } } @@ -348,6 +369,7 @@ impl From for Error { mod tests { use super::*; use core::mem::size_of; + use std::collections::BTreeMap; #[test] fn test_primitive_u8() -> Result<()> { @@ -515,4 +537,81 @@ mod tests { assert_eq!(out, expected); Ok(()) } + + #[test] + fn test_struct_simple() -> Result<()> { + #[derive(Serialize, Encode)] + struct Foo { + a: Bar, + b: Option, + } + #[derive(Serialize, Encode)] + struct Bar(u8); + #[derive(Serialize, Encode)] + struct Baz(String, u16); + + let input = Foo { + a: Bar(0xFF), + b: Some(Baz("lol".into(), u16::MAX)), + }; + let mut out = Vec::::new(); + let expected = input.encode(); + + to_writer(&mut out, &input)?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_vec_simple() -> Result<()> { + let input: Vec = vec!["hello".into(), "beautiful".into(), "people".into()]; + let mut out = Vec::::new(); + let expected = input.encode(); + + to_writer(&mut out, &input)?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_struct_mix() -> Result<()> { + #[derive(Serialize, Encode)] + struct Foo<'a> { + a: Vec, + b: (Bar<'a>, Bar<'a>, Bar<'a>), + } + #[derive(Serialize, Encode)] + enum Bar<'a> { + A { thing: &'a str }, + B(Baz), + C(BTreeMap, i64), + } + #[derive(Serialize, Encode)] + struct Baz; + + let input = Foo { + a: vec!["hello".into(), "beautiful".into(), "people".into()], + b: ( + Bar::A { thing: "barbarbar" }, + Bar::B(Baz), + Bar::C( + { + let mut h = BTreeMap::new(); + h.insert("key".into(), false); + h + }, + i64::MIN, + ), + ), + }; + let mut out = Vec::::new(); + let expected = input.encode(); + + to_writer(&mut out, &input)?; + + assert_eq!(out, expected); + Ok(()) + } } From 7ca49f9782ff27642d61329fbe19c3ea36d55059 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 9 Sep 2021 15:53:52 +0200 Subject: [PATCH 21/59] Fix clippy warnings --- src/serializer.rs | 27 ++++++++++++++++++--------- src/value.rs | 8 ++++---- 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index 90083fb..16685fc 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -39,23 +39,28 @@ where type SerializeStructVariant = Self; fn serialize_bool(self, v: bool) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_i8(self, v: i8) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_i16(self, v: i16) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_i32(self, v: i32) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_i64(self, v: i64) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_u8(self, v: u8) -> Result { @@ -63,15 +68,18 @@ where } fn serialize_u16(self, v: u16) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_u32(self, v: u32) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_u64(self, v: u64) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_f32(self, _v: f32) -> Result { @@ -87,7 +95,8 @@ where } fn serialize_str(self, v: &str) -> Result { - Ok(v.encode_to(&mut self.0)) + v.encode_to(&mut self.0); + Ok(()) } fn serialize_bytes(self, v: &[u8]) -> Result { diff --git a/src/value.rs b/src/value.rs index a897c23..e64ae2f 100644 --- a/src/value.rs +++ b/src/value.rs @@ -38,7 +38,7 @@ impl<'a> Serialize for Value<'a> { match self.ty.type_def() { TypeDef::Composite(cmp) => { let fields = cmp.fields(); - if fields.len() == 0 { + if fields.is_empty() { ser.serialize_unit_struct(name) } else if is_tuple(fields) { let mut state = ser.serialize_tuple_struct(name, fields.len())?; @@ -47,7 +47,7 @@ impl<'a> Serialize for Value<'a> { } state.end() } else { - let mut state = ser.serialize_struct(&name, fields.len())?; + let mut state = ser.serialize_struct(name, fields.len())?; for f in fields { state.serialize_field( f.name().unwrap(), @@ -162,7 +162,7 @@ impl<'a> Value<'a> { } fn sub_value(&self, ty: Type) -> Value { - let size = ty_data_size(&ty, &self.remaining_data()); + let size = ty_data_size(&ty, self.remaining_data()); Value::new(self.extract_data(size), ty) } @@ -185,7 +185,7 @@ fn is_tuple(fields: &[Field]) -> bool { fields.first().and_then(Field::name).is_none() } -fn ty_data_size<'a>(ty: &Type, data: &'a [u8]) -> usize { +fn ty_data_size(ty: &Type, data: &[u8]) -> usize { match ty.type_def() { TypeDef::Primitive(p) => match p { Primitive::U8 => mem::size_of::(), From 32a1038a03c5a0c760460e056f8a9c3588293c37 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Sat, 11 Sep 2021 08:34:00 +0200 Subject: [PATCH 22/59] Serialize `Value` that are maps(BTreeMap) --- src/value.rs | 58 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/value.rs b/src/value.rs index e64ae2f..e161b7a 100644 --- a/src/value.rs +++ b/src/value.rs @@ -2,16 +2,19 @@ use byteorder::{ByteOrder, LE}; use core::cell::Cell; use core::convert::TryInto; use core::str; -use scale_info::{prelude::*, Field, Type, TypeDef, TypeDefPrimitive as Primitive}; +use scale_info::{ + prelude::*, Field, Type, TypeDef, TypeDefComposite, TypeDefPrimitive as Primitive, +}; use serde::ser::{ - SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, SerializeTupleStruct, - SerializeTupleVariant, + SerializeMap, SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, + SerializeTupleStruct, SerializeTupleVariant, }; use serde::Serialize; /// A container for SCALE encoded data that can serialize types /// directly with the help of a type registry and without using an /// intermediate representation that requires allocating data. +#[derive(Debug)] pub struct Value<'a> { data: &'a [u8], ty: Type, @@ -40,6 +43,18 @@ impl<'a> Serialize for Value<'a> { let fields = cmp.fields(); if fields.is_empty() { ser.serialize_unit_struct(name) + } else if is_map(&self.ty) { + let (len, p_size) = sequence_len(self.remaining_data()); + self.advance_idx(p_size); + + let mut state = ser.serialize_map(Some(len))?; + let (kty, vty) = map_types(&cmp); + for _ in 0..len { + let key = &self.sub_value(kty.clone()); + let val = &self.sub_value(vty.clone()); + state.serialize_entry(key, val)?; + } + state.end() } else if is_tuple(fields) { let mut state = ser.serialize_tuple_struct(name, fields.len())?; for f in fields { @@ -181,6 +196,23 @@ impl<'a> Value<'a> { } } +fn is_map(ty: &Type) -> bool { + ty.path().segments() == ["BTreeMap"] +} +fn map_types(ty: &TypeDefComposite) -> (Type, Type) { + let field = ty.fields().first().expect("map"); + // Type information of BTreeMap is weirdly packed + if let TypeDef::Sequence(s) = field.ty().type_info().type_def() { + if let TypeDef::Tuple(t) = s.type_param().type_info().type_def() { + assert_eq!(t.fields().len(), 2); + let key_ty = t.fields().first().expect("key").type_info(); + let val_ty = t.fields().last().expect("val").type_info(); + return (key_ty, val_ty); + } + } + unreachable!() +} + fn is_tuple(fields: &[Field]) -> bool { fields.first().and_then(Field::name).is_none() } @@ -237,6 +269,7 @@ fn ty_data_size(ty: &Type, data: &[u8]) -> usize { } } +#[inline] fn compact_len(data: &[u8]) -> usize { match data[0] % 0b100 { 0 => 1, @@ -267,6 +300,8 @@ fn sequence_len(data: &[u8]) -> (usize, usize) { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; use anyhow::Error; use codec::Encode; @@ -468,6 +503,23 @@ mod tests { Ok(()) } + #[test] + fn serialize_map() -> Result<(), Error> { + let value = { + let mut m = BTreeMap::::new(); + m.insert("foo".into(), i32::MAX); + m.insert("bar".into(), i32::MIN); + m + }; + + let data = value.encode(); + let info = BTreeMap::::type_info(); + let val = Value::new(&data, info); + + assert_eq!(to_value(val)?, to_value(value)?); + Ok(()) + } + #[test] fn serialize_complex_struct_with_enum() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] From 50344e42136a4da2bef345d36f6007115741b25f Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Sun, 12 Sep 2021 09:40:54 +0200 Subject: [PATCH 23/59] Add SerdeType that maps scale-info types to serde data model --- src/lib.rs | 137 +++++++++++++++++++++++++++++ src/value.rs | 242 +++++++++++++++++++++------------------------------ 2 files changed, 237 insertions(+), 142 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index fccfbc2..d8d5d0c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,3 +11,140 @@ mod value; #[cfg(feature = "serializer")] pub use serializer::{to_writer, Serializer}; pub use value::Value; + +use scale_info::{Field, MetaType, Type, Variant}; + +#[rustfmt::skip] +/// A convenient representation of the types that matches serde model more closely +#[derive(Debug)] +enum SerdeType<'a> { + Bool, + U8, U16, U32, U64, U128, + I8, I16, I32, I64, I128, + Bytes, + Char, + Str, + Sequence(Type), + Map(Type, Type), + Tuple(TupleOrArray<'a>), + OptionNone, OptionSome(Type), + VariantUnit(&'a Variant), VariantNewType(&'a Variant), + VariantTuple(&'a Variant), VariantStruct(&'a Variant), + Struct(&'a [Field]), StructUnit, StructNewType, StructTuple(&'a [Field]), +} + +#[derive(Debug)] +enum TupleOrArray<'a> { + Array(&'a MetaType, u32), + Tuple(&'a [MetaType]), +} +impl<'a> TupleOrArray<'a> { + fn len(&self) -> usize { + match self { + Self::Array(_, len) => *len as usize, + Self::Tuple(fields) => fields.len(), + } + } + + fn type_info(&self, i: usize) -> Type { + match self { + Self::Array(ty, _) => ty.type_info(), + Self::Tuple(fields) => fields[i].type_info(), + } + } +} + +impl<'a> SerdeType<'a> { + fn from(ty: &'a Type, maybe_variant_index: u8) -> Self { + use scale_info::{TypeDef, TypeDef::*, TypeDefComposite, TypeDefPrimitive}; + #[inline] + fn is_map(ty: &Type) -> bool { + ty.path().segments() == ["BTreeMap"] + } + fn map_types(ty: &TypeDefComposite) -> (Type, Type) { + let field = ty.fields().first().expect("map"); + // Type information of BTreeMap is weirdly packed + if let TypeDef::Sequence(s) = field.ty().type_info().type_def() { + if let TypeDef::Tuple(t) = s.type_param().type_info().type_def() { + assert_eq!(t.fields().len(), 2); + let key_ty = t.fields().first().expect("key").type_info(); + let val_ty = t.fields().last().expect("val").type_info(); + return (key_ty, val_ty); + } + } + unreachable!() + } + #[inline] + fn is_tuple(fields: &[Field]) -> bool { + fields.first().and_then(Field::name).is_none() + } + + let name = ty.path().segments().last().copied().unwrap_or(""); + + match ty.type_def() { + Composite(c) => { + let fields = c.fields(); + if fields.is_empty() { + Self::StructUnit + } else if is_map(ty) { + let (k, v) = map_types(c); + Self::Map(k, v) + } else if is_tuple(fields) { + Self::StructTuple(fields) + } else { + Self::Struct(fields) + } + } + Variant(enu) => { + let variant = enu + .variants() + .iter() + .find(|v| v.index() == maybe_variant_index) + .expect("variant"); + let fields = variant.fields(); + if fields.is_empty() { + if name == "Option" && variant.name() == &"None" { + Self::OptionNone + } else { + Self::VariantUnit(variant) + } + } else if is_tuple(fields) { + if fields.len() == 1 { + let ty = fields.first().unwrap().ty().type_info(); + return if name == "Option" && variant.name() == &"Some" { + Self::OptionSome(ty) + } else { + Self::VariantNewType(variant) + }; + } else { + Self::VariantTuple(variant) + } + } else { + Self::VariantStruct(variant) + } + } + Sequence(s) => Self::Sequence(s.type_param().type_info()), + Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param(), a.len())), + Tuple(t) => Self::Tuple(TupleOrArray::Tuple(t.fields())), + Primitive(p) => match p { + TypeDefPrimitive::U8 => Self::U8, + TypeDefPrimitive::U16 => Self::U16, + TypeDefPrimitive::U32 => Self::U32, + TypeDefPrimitive::U64 => Self::U64, + TypeDefPrimitive::U128 => Self::U128, + TypeDefPrimitive::I8 => Self::I8, + TypeDefPrimitive::I16 => Self::I16, + TypeDefPrimitive::I32 => Self::I32, + TypeDefPrimitive::I64 => Self::I64, + TypeDefPrimitive::I128 => Self::I128, + TypeDefPrimitive::Bool => Self::Bool, + TypeDefPrimitive::Str => Self::Str, + TypeDefPrimitive::Char => Self::Char, + TypeDefPrimitive::U256 => unimplemented!(), + TypeDefPrimitive::I256 => unimplemented!(), + }, + Compact(_c) => todo!(), + BitSequence(_b) => todo!(), + } + } +} diff --git a/src/value.rs b/src/value.rs index e161b7a..a1e07d0 100644 --- a/src/value.rs +++ b/src/value.rs @@ -2,15 +2,15 @@ use byteorder::{ByteOrder, LE}; use core::cell::Cell; use core::convert::TryInto; use core::str; -use scale_info::{ - prelude::*, Field, Type, TypeDef, TypeDefComposite, TypeDefPrimitive as Primitive, -}; +use scale_info::{prelude::*, Type, TypeDef, TypeDefPrimitive as Primitive}; use serde::ser::{ SerializeMap, SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, SerializeTupleStruct, SerializeTupleVariant, }; use serde::Serialize; +use crate::SerdeType; + /// A container for SCALE encoded data that can serialize types /// directly with the help of a type registry and without using an /// intermediate representation that requires allocating data. @@ -37,152 +37,131 @@ impl<'a> Serialize for Value<'a> { S: serde::Serializer, { let name = self.ty_name(); - - match self.ty.type_def() { - TypeDef::Composite(cmp) => { - let fields = cmp.fields(); - if fields.is_empty() { - ser.serialize_unit_struct(name) - } else if is_map(&self.ty) { - let (len, p_size) = sequence_len(self.remaining_data()); - self.advance_idx(p_size); - - let mut state = ser.serialize_map(Some(len))?; - let (kty, vty) = map_types(&cmp); - for _ in 0..len { - let key = &self.sub_value(kty.clone()); - let val = &self.sub_value(vty.clone()); - state.serialize_entry(key, val)?; - } - state.end() - } else if is_tuple(fields) { - let mut state = ser.serialize_tuple_struct(name, fields.len())?; - for f in fields { - state.serialize_field(&self.sub_value(f.ty().type_info()))?; - } - state.end() - } else { - let mut state = ser.serialize_struct(name, fields.len())?; - for f in fields { - state.serialize_field( - f.name().unwrap(), - &self.sub_value(f.ty().type_info()), - )?; - } - state.end() - } + let data = self.data_left(); + let ty: SerdeType = SerdeType::from(&self.ty, *data.get(0).unwrap_or(&0)); + + use SerdeType::*; + match ty { + Bool => ser.serialize_bool(data[0] != 0), + U8 => ser.serialize_u8(data[0]), + U16 => ser.serialize_u16(LE::read_u16(data)), + U32 => ser.serialize_u32(LE::read_u32(data)), + U64 => ser.serialize_u64(LE::read_u64(data)), + U128 => ser.serialize_u128(LE::read_u128(data)), + I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), + I16 => ser.serialize_i16(LE::read_i16(data)), + I32 => ser.serialize_i32(LE::read_i32(data)), + I64 => ser.serialize_i64(LE::read_i64(data)), + I128 => ser.serialize_i128(LE::read_i128(data)), + Bytes => unimplemented!(), + Char => { + let n = LE::read_u32(data); + ser.serialize_char(char::from_u32(n).unwrap()) } - TypeDef::Variant(enu) => { - let idx = self.extract_data(1)[0]; - let var = enu - .variants() - .iter() - .find(|v| v.index() == idx) - .expect("variant"); - - let fields = var.fields(); - if fields.is_empty() { - if name == "Option" && *var.name() == "None" { - return ser.serialize_none(); - } - ser.serialize_unit_variant(name, var.index().into(), var.name()) - } else if is_tuple(fields) { - if fields.len() == 1 { - let ty = fields.first().unwrap().ty().type_info(); - let val = self.sub_value(ty); - - if name == "Option" && *var.name() == "Some" { - return ser.serialize_some(&val); - } - return ser.serialize_newtype_variant(name, idx.into(), var.name(), &val); - } - let mut s = - ser.serialize_tuple_variant(name, idx.into(), var.name(), fields.len())?; - for f in var.fields() { - s.serialize_field(&self.sub_value(f.ty().type_info()))?; - } - s.end() - } else { - let mut s = ser.serialize_struct_variant( - name, - var.index().into(), - var.name(), - fields.len(), - )?; - for f in var.fields() { - s.serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; - } - s.end() - } + Str => { + let (_, s) = sequence_len(data); + self.advance_idx(s); + ser.serialize_str(str::from_utf8(self.data_left()).unwrap()) } - TypeDef::Sequence(seq) => { - let ty = seq.type_param().type_info(); - let (len, p_size) = sequence_len(self.remaining_data()); + Sequence(ty) => { + let (len, p_size) = sequence_len(self.data_left()); self.advance_idx(p_size); - let mut seq = ser.serialize_seq(Some(len))?; + let mut seq = ser.serialize_seq(Some(len))?; for _ in 0..len { seq.serialize_element(&self.sub_value(ty.clone()))?; } seq.end() } - TypeDef::Array(arr) => { - let mut s = ser.serialize_tuple(arr.len().try_into().unwrap())?; - let ty = arr.type_param().type_info(); - for _ in 0..arr.len() { - s.serialize_element(&self.sub_value(ty.clone()))?; + Map(ty_k, ty_v) => { + let (len, p_size) = sequence_len(self.data_left()); + self.advance_idx(p_size); + + let mut state = ser.serialize_map(Some(len))?; + for _ in 0..len { + let key = &self.sub_value(ty_k.clone()); + let val = &self.sub_value(ty_v.clone()); + state.serialize_entry(key, val)?; } - s.end() + state.end() + } + Tuple(t) => { + let mut state = ser.serialize_tuple(t.len())?; + for i in 0..t.len() { + state.serialize_element(&self.sub_value(t.type_info(i)))?; + } + state.end() } - TypeDef::Tuple(tup) => { - let mut state = ser.serialize_tuple(tup.fields().len())?; - for f in tup.fields() { - state.serialize_element(&self.sub_value(f.type_info()))?; + Struct(fields) => { + let mut state = ser.serialize_struct(name, fields.len())?; + for f in fields { + state + .serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; } state.end() } - TypeDef::Primitive(prm) => match prm { - Primitive::U8 => ser.serialize_u8(self.remaining_data()[0]), - Primitive::U16 => ser.serialize_u16(LE::read_u16(self.remaining_data())), - Primitive::U32 => ser.serialize_u32(LE::read_u32(self.remaining_data())), - Primitive::U64 => ser.serialize_u64(LE::read_u64(self.remaining_data())), - Primitive::U128 => ser.serialize_u128(LE::read_u128(self.remaining_data())), - Primitive::I8 => ser.serialize_i8(i8::from_le_bytes([self.remaining_data()[0]])), - Primitive::I16 => ser.serialize_i16(LE::read_i16(self.remaining_data())), - Primitive::I32 => ser.serialize_i32(LE::read_i32(self.remaining_data())), - Primitive::I64 => ser.serialize_i64(LE::read_i64(self.remaining_data())), - Primitive::I128 => ser.serialize_i128(LE::read_i128(self.remaining_data())), - Primitive::Bool => ser.serialize_bool(self.remaining_data()[0] != 0), - Primitive::Str => { - let (_, s) = sequence_len(self.remaining_data()); - self.advance_idx(s); - ser.serialize_str(str::from_utf8(self.remaining_data()).unwrap()) + StructUnit => ser.serialize_unit_struct(name), + StructNewType => unimplemented!(), + StructTuple(fields) => { + let mut state = ser.serialize_tuple_struct(name, fields.len())?; + for f in fields { + state.serialize_field(&self.sub_value(f.ty().type_info()))?; } - Primitive::Char => { - let n = LE::read_u32(self.remaining_data()); - ser.serialize_char(char::from_u32(n).unwrap()) + state.end() + } + OptionNone => ser.serialize_none(), + OptionSome(ty) => { + self.advance_idx(1); + ser.serialize_some(&self.sub_value(ty)) + } + VariantUnit(v) => ser.serialize_unit_variant(name, v.index().into(), v.name()), + VariantNewType(v) => { + self.advance_idx(1); + let ty = v.fields().first().unwrap().ty().type_info(); + ser.serialize_newtype_variant(name, v.index().into(), v.name(), &self.sub_value(ty)) + } + VariantTuple(v) => { + self.advance_idx(1); + let mut s = ser.serialize_tuple_variant( + name, + v.index().into(), + v.name(), + v.fields().len(), + )?; + for f in v.fields() { + s.serialize_field(&self.sub_value(f.ty().type_info()))?; } - Primitive::U256 => unimplemented!(), - Primitive::I256 => unimplemented!(), - }, - TypeDef::Compact(_) => todo!(), - TypeDef::BitSequence(_) => todo!(), + s.end() + } + VariantStruct(v) => { + self.advance_idx(1); + let mut s = ser.serialize_struct_variant( + name, + v.index().into(), + v.name(), + v.fields().len(), + )?; + for f in v.fields() { + s.serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; + } + s.end() + } } } } impl<'a> Value<'a> { - fn remaining_data(&self) -> &[u8] { + fn data_left(&self) -> &[u8] { &self.data[self.idx.get()..] } fn sub_value(&self, ty: Type) -> Value { - let size = ty_data_size(&ty, self.remaining_data()); + let size = ty_data_size(&ty, self.data_left()); Value::new(self.extract_data(size), ty) } fn extract_data(&self, end: usize) -> &[u8] { - let (data, _) = &self.remaining_data().split_at(end); + let (data, _) = &self.data_left().split_at(end); self.advance_idx(end); data } @@ -196,27 +175,6 @@ impl<'a> Value<'a> { } } -fn is_map(ty: &Type) -> bool { - ty.path().segments() == ["BTreeMap"] -} -fn map_types(ty: &TypeDefComposite) -> (Type, Type) { - let field = ty.fields().first().expect("map"); - // Type information of BTreeMap is weirdly packed - if let TypeDef::Sequence(s) = field.ty().type_info().type_def() { - if let TypeDef::Tuple(t) = s.type_param().type_info().type_def() { - assert_eq!(t.fields().len(), 2); - let key_ty = t.fields().first().expect("key").type_info(); - let val_ty = t.fields().last().expect("val").type_info(); - return (key_ty, val_ty); - } - } - unreachable!() -} - -fn is_tuple(fields: &[Field]) -> bool { - fields.first().and_then(Field::name).is_none() -} - fn ty_data_size(ty: &Type, data: &[u8]) -> usize { match ty.type_def() { TypeDef::Primitive(p) => match p { @@ -265,7 +223,7 @@ fn ty_data_size(ty: &Type, data: &[u8]) -> usize { TypeDef::Array(a) => a.len().try_into().unwrap(), TypeDef::Tuple(t) => t.fields().len(), TypeDef::Compact(_) => compact_len(data), - TypeDef::BitSequence(_) => todo!(), + TypeDef::BitSequence(_) => unimplemented!(), } } From 3a41655ff32ce981b2a336fd4a4d16421326eb66 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Tue, 14 Sep 2021 10:30:38 +0200 Subject: [PATCH 24/59] Separate SerdeType::Variant to not require knowning specific variant in advance --- src/lib.rs | 142 +++++++++++++++++++++++++++++---------------------- src/value.rs | 85 ++++++++++++++++-------------- 2 files changed, 127 insertions(+), 100 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d8d5d0c..d015bef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,8 +14,9 @@ pub use value::Value; use scale_info::{Field, MetaType, Type, Variant}; +/// A convenient representation of the scale-info types to a format +/// that matches serde model more closely #[rustfmt::skip] -/// A convenient representation of the types that matches serde model more closely #[derive(Debug)] enum SerdeType<'a> { Bool, @@ -27,36 +28,15 @@ enum SerdeType<'a> { Sequence(Type), Map(Type, Type), Tuple(TupleOrArray<'a>), - OptionNone, OptionSome(Type), - VariantUnit(&'a Variant), VariantNewType(&'a Variant), - VariantTuple(&'a Variant), VariantStruct(&'a Variant), Struct(&'a [Field]), StructUnit, StructNewType, StructTuple(&'a [Field]), + Variant(&'a str, &'a [Variant]), } -#[derive(Debug)] -enum TupleOrArray<'a> { - Array(&'a MetaType, u32), - Tuple(&'a [MetaType]), -} -impl<'a> TupleOrArray<'a> { - fn len(&self) -> usize { - match self { - Self::Array(_, len) => *len as usize, - Self::Tuple(fields) => fields.len(), - } - } - - fn type_info(&self, i: usize) -> Type { - match self { - Self::Array(ty, _) => ty.type_info(), - Self::Tuple(fields) => fields[i].type_info(), - } - } -} - -impl<'a> SerdeType<'a> { - fn from(ty: &'a Type, maybe_variant_index: u8) -> Self { +impl<'a> From<&'a Type> for SerdeType<'a> { + fn from(ty: &'a Type) -> Self { use scale_info::{TypeDef, TypeDef::*, TypeDefComposite, TypeDefPrimitive}; + let name = ty.path().segments().last().copied().unwrap_or(""); + #[inline] fn is_map(ty: &Type) -> bool { ty.path().segments() == ["BTreeMap"] @@ -74,19 +54,13 @@ impl<'a> SerdeType<'a> { } unreachable!() } - #[inline] - fn is_tuple(fields: &[Field]) -> bool { - fields.first().and_then(Field::name).is_none() - } - - let name = ty.path().segments().last().copied().unwrap_or(""); match ty.type_def() { Composite(c) => { let fields = c.fields(); if fields.is_empty() { Self::StructUnit - } else if is_map(ty) { + } else if is_map(&ty) { let (k, v) = map_types(c); Self::Map(k, v) } else if is_tuple(fields) { @@ -95,34 +69,7 @@ impl<'a> SerdeType<'a> { Self::Struct(fields) } } - Variant(enu) => { - let variant = enu - .variants() - .iter() - .find(|v| v.index() == maybe_variant_index) - .expect("variant"); - let fields = variant.fields(); - if fields.is_empty() { - if name == "Option" && variant.name() == &"None" { - Self::OptionNone - } else { - Self::VariantUnit(variant) - } - } else if is_tuple(fields) { - if fields.len() == 1 { - let ty = fields.first().unwrap().ty().type_info(); - return if name == "Option" && variant.name() == &"Some" { - Self::OptionSome(ty) - } else { - Self::VariantNewType(variant) - }; - } else { - Self::VariantTuple(variant) - } - } else { - Self::VariantStruct(variant) - } - } + Variant(v) => Self::Variant(name, v.variants()), Sequence(s) => Self::Sequence(s.type_param().type_info()), Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param(), a.len())), Tuple(t) => Self::Tuple(TupleOrArray::Tuple(t.fields())), @@ -148,3 +95,74 @@ impl<'a> SerdeType<'a> { } } } + +impl<'a> SerdeType<'a> { + fn pick_variant(self, index: u8) -> EnumVariant<'a> { + match self { + SerdeType::Variant(name, variants) => { + let variant = variants + .iter() + .find(|v| v.index() == index) + .expect("variant"); + let fields = variant.fields(); + if fields.is_empty() { + if name == "Option" && variant.name() == &"None" { + EnumVariant::OptionNone.into() + } else { + EnumVariant::Unit(variant).into() + } + } else if is_tuple(fields) { + if fields.len() == 1 { + let ty = fields.first().unwrap().ty().type_info(); + return if name == "Option" && variant.name() == &"Some" { + EnumVariant::OptionSome(ty).into() + } else { + EnumVariant::NewType(variant).into() + }; + } else { + EnumVariant::Tuple(variant).into() + } + } else { + EnumVariant::Struct(variant).into() + } + } + _ => unreachable!(), + } + } +} + +#[derive(Debug)] +enum EnumVariant<'a> { + OptionNone, + OptionSome(Type), + Unit(&'a Variant), + NewType(&'a Variant), + Tuple(&'a Variant), + Struct(&'a Variant), +} + +#[derive(Debug)] +enum TupleOrArray<'a> { + Array(&'a MetaType, u32), + Tuple(&'a [MetaType]), +} +impl<'a> TupleOrArray<'a> { + fn len(&self) -> usize { + match self { + Self::Array(_, len) => *len as usize, + Self::Tuple(fields) => fields.len(), + } + } + + fn type_info(&self, i: usize) -> Type { + match self { + Self::Array(ty, _) => ty.type_info(), + Self::Tuple(fields) => fields[i].type_info(), + } + } +} + +#[inline] +fn is_tuple(fields: &[Field]) -> bool { + fields.first().and_then(Field::name).is_none() +} diff --git a/src/value.rs b/src/value.rs index a1e07d0..c26dee2 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,3 +1,4 @@ +use crate::{EnumVariant, SerdeType}; use byteorder::{ByteOrder, LE}; use core::cell::Cell; use core::convert::TryInto; @@ -9,8 +10,6 @@ use serde::ser::{ }; use serde::Serialize; -use crate::SerdeType; - /// A container for SCALE encoded data that can serialize types /// directly with the help of a type registry and without using an /// intermediate representation that requires allocating data. @@ -38,7 +37,7 @@ impl<'a> Serialize for Value<'a> { { let name = self.ty_name(); let data = self.data_left(); - let ty: SerdeType = SerdeType::from(&self.ty, *data.get(0).unwrap_or(&0)); + let ty = SerdeType::from(&self.ty); use SerdeType::*; match ty { @@ -109,43 +108,53 @@ impl<'a> Serialize for Value<'a> { } state.end() } - OptionNone => ser.serialize_none(), - OptionSome(ty) => { - self.advance_idx(1); - ser.serialize_some(&self.sub_value(ty)) - } - VariantUnit(v) => ser.serialize_unit_variant(name, v.index().into(), v.name()), - VariantNewType(v) => { - self.advance_idx(1); - let ty = v.fields().first().unwrap().ty().type_info(); - ser.serialize_newtype_variant(name, v.index().into(), v.name(), &self.sub_value(ty)) - } - VariantTuple(v) => { - self.advance_idx(1); - let mut s = ser.serialize_tuple_variant( - name, - v.index().into(), - v.name(), - v.fields().len(), - )?; - for f in v.fields() { - s.serialize_field(&self.sub_value(f.ty().type_info()))?; + Variant(_, _) => match ty.pick_variant(data[0]) { + EnumVariant::OptionNone => ser.serialize_none(), + EnumVariant::OptionSome(ty) => { + self.advance_idx(1); + ser.serialize_some(&self.sub_value(ty)) } - s.end() - } - VariantStruct(v) => { - self.advance_idx(1); - let mut s = ser.serialize_struct_variant( - name, - v.index().into(), - v.name(), - v.fields().len(), - )?; - for f in v.fields() { - s.serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; + EnumVariant::Unit(v) => { + ser.serialize_unit_variant(name, v.index().into(), v.name()) } - s.end() - } + EnumVariant::NewType(v) => { + self.advance_idx(1); + let ty = v.fields().first().unwrap().ty().type_info(); + ser.serialize_newtype_variant( + name, + v.index().into(), + v.name(), + &self.sub_value(ty), + ) + } + + EnumVariant::Tuple(v) => { + self.advance_idx(1); + let mut s = ser.serialize_tuple_variant( + name, + v.index().into(), + v.name(), + v.fields().len(), + )?; + for f in v.fields() { + s.serialize_field(&self.sub_value(f.ty().type_info()))?; + } + s.end() + } + EnumVariant::Struct(v) => { + self.advance_idx(1); + let mut s = ser.serialize_struct_variant( + name, + v.index().into(), + v.name(), + v.fields().len(), + )?; + for f in v.fields() { + s.serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; + } + s.end() + } + }, } } } From a45ab018e01c60a10f905871ad8965f50ea6456d Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 15 Sep 2021 20:10:47 +0200 Subject: [PATCH 25/59] Handle newtype structs --- src/lib.rs | 4 +++- src/value.rs | 16 +++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index d015bef..2c0784c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -28,7 +28,7 @@ enum SerdeType<'a> { Sequence(Type), Map(Type, Type), Tuple(TupleOrArray<'a>), - Struct(&'a [Field]), StructUnit, StructNewType, StructTuple(&'a [Field]), + Struct(&'a [Field]), StructUnit, StructNewType(Type), StructTuple(&'a [Field]), Variant(&'a str, &'a [Variant]), } @@ -63,6 +63,8 @@ impl<'a> From<&'a Type> for SerdeType<'a> { } else if is_map(&ty) { let (k, v) = map_types(c); Self::Map(k, v) + } else if fields.len() == 1 { + Self::StructNewType(fields.first().unwrap().ty().type_info()) } else if is_tuple(fields) { Self::StructTuple(fields) } else { diff --git a/src/value.rs b/src/value.rs index c26dee2..c2d4d6a 100644 --- a/src/value.rs +++ b/src/value.rs @@ -100,7 +100,7 @@ impl<'a> Serialize for Value<'a> { state.end() } StructUnit => ser.serialize_unit_struct(name), - StructNewType => unimplemented!(), + StructNewType(ty) => ser.serialize_newtype_struct(name, &self.sub_value(ty)), StructTuple(fields) => { let mut state = ser.serialize_tuple_struct(name, fields.len())?; for f in fields { @@ -495,19 +495,21 @@ mod tests { That(i16), } #[derive(Encode, Serialize, TypeInfo)] + struct Baz(String); + #[derive(Encode, Serialize, TypeInfo)] struct Foo { bar: Vec, - baz: Option, + baz: Option, } - let extract_value = Foo { + let expected = Foo { bar: [Bar::That(i16::MAX), Bar::This].into(), - baz: Some("aliquam malesuada bibendum arcu vitae".into()), + baz: Some(Baz("aliquam malesuada bibendum arcu vitae".into())), }; - let data = extract_value.encode(); + let data = expected.encode(); let info = Foo::type_info(); - let val = Value::new(&data, info); + let out = Value::new(&data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + assert_eq!(to_value(out)?, to_value(expected)?); Ok(()) } From 6775da0636d02907ea7f45ab66936562d72e7c25 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 15 Sep 2021 20:11:39 +0200 Subject: [PATCH 26/59] WIP Serializer: serialize simple json value --- src/serializer.rs | 293 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 241 insertions(+), 52 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index 16685fc..2b00cf5 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,25 +1,50 @@ use codec::{Compact, Encode}; use core::fmt; use core2::io; +use scale_info::{Type, TypeInfo}; use serde::{ser, Serialize}; +use crate::SerdeType; + type Result = core::result::Result; +#[derive(TypeInfo)] +struct Noop; + pub fn to_writer(writer: W, value: &T) -> Result<()> where T: Serialize, W: io::Write, { - let mut serializer = Serializer::new(writer); + to_writer_with_info(writer, value, Noop::type_info()) +} + +pub fn to_writer_with_info(writer: W, value: &T, info: Type) -> Result<()> +where + T: Serialize, + W: io::Write, +{ + let info = if info.path().segments().eq(&["scales", "serializer", "Noop"]) { + None + } else { + Some(info) + }; + let mut serializer = Serializer::new(writer, info); value.serialize(&mut serializer)?; Ok(()) } -pub struct Serializer(W); +pub struct Serializer { + out: W, + ty: Option, +} -impl Serializer { - pub fn new(writer: W) -> Self { - Serializer(writer) +impl Serializer +where + W: io::Write, +{ + pub fn new(out: W, ty: Option) -> Self { + Serializer { out, ty } } } @@ -30,55 +55,65 @@ where type Ok = (); type Error = Error; - type SerializeSeq = Self; - type SerializeTuple = Self; - type SerializeTupleStruct = Self; - type SerializeTupleVariant = Self; - type SerializeMap = Self; - type SerializeStruct = Self; - type SerializeStructVariant = Self; + type SerializeSeq = TypedSerializer<'a, W>; + type SerializeTuple = TypedSerializer<'a, W>; + type SerializeTupleStruct = TypedSerializer<'a, W>; + type SerializeTupleVariant = TypedSerializer<'a, W>; + type SerializeMap = TypedSerializer<'a, W>; + type SerializeStruct = TypedSerializer<'a, W>; + type SerializeStructVariant = TypedSerializer<'a, W>; fn serialize_bool(self, v: bool) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_i8(self, v: i8) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_i16(self, v: i16) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_i32(self, v: i32) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_i64(self, v: i64) -> Result { - v.encode_to(&mut self.0); + match self.ty() { + Some(SerdeType::I8) => (v as i8).encode_to(&mut self.out), + Some(SerdeType::I16) => (v as i16).encode_to(&mut self.out), + Some(SerdeType::I32) => (v as i32).encode_to(&mut self.out), + _ => v.encode_to(&mut self.out), + } Ok(()) } fn serialize_u8(self, v: u8) -> Result { - self.0.write_all(&[v]).map_err(Error::from) + self.out.write_all(&[v]).map_err(Error::from) } fn serialize_u16(self, v: u16) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_u32(self, v: u32) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_u64(self, v: u64) -> Result { - v.encode_to(&mut self.0); + match self.ty() { + Some(SerdeType::U8) => (v as u8).encode_to(&mut self.out), + Some(SerdeType::U16) => (v as u16).encode_to(&mut self.out), + Some(SerdeType::U32) => (v as u32).encode_to(&mut self.out), + _ => v.encode_to(&mut self.out), + } Ok(()) } @@ -95,31 +130,33 @@ where } fn serialize_str(self, v: &str) -> Result { - v.encode_to(&mut self.0); + v.encode_to(&mut self.out); Ok(()) } fn serialize_bytes(self, v: &[u8]) -> Result { - self.0.write_all(v).map_err(Error::from) + self.out.write_all(v).map_err(Error::from) } fn serialize_none(self) -> Result { - self.0.write_all(&[0x00]).map_err(Error::from) + self.out.write_all(&[0x00]).map_err(Error::from) } fn serialize_some(self, value: &T) -> Result where T: Serialize, { - self.0.write_all(&[0x01])?; + self.out.write_all(&[0x01])?; value.serialize(self) } fn serialize_unit(self) -> Result { + println!("===== u {:?}\n", self.ty); Ok(()) } fn serialize_unit_struct(self, _name: &'static str) -> Result { + println!("===== us {:?}\n", self.ty); Ok(()) } @@ -129,6 +166,7 @@ where variant_index: u32, _variant: &'static str, ) -> Result { + println!("===== uv {:?}\n", self.ty); (variant_index as u8).serialize(self) } @@ -136,6 +174,7 @@ where where T: Serialize, { + println!("===== ns {:?}\n", self.ty); value.serialize(self) } @@ -149,17 +188,23 @@ where where T: Serialize, { - self.0.write_all(&[variant_index as u8])?; + println!("===== nv {:?}\n", self.ty); + self.out.write_all(&[variant_index as u8])?; value.serialize(self) } fn serialize_seq(self, len: Option) -> Result { - Compact(len.expect("known length") as u64).encode_to(&mut self.0); - Ok(self) + self.maybe_some()?; + println!("===== seq {:?}\n", self.ty); + if !matches!(self.ty(), Some(SerdeType::StructTuple(_))) { + Compact(len.expect("known length") as u64).encode_to(&mut self.out); + } + Ok(self.into()) } fn serialize_tuple(self, _len: usize) -> Result { - Ok(self) + println!("===== tup {:?}\n", self.ty); + Ok(self.into()) } fn serialize_tuple_struct( @@ -167,7 +212,8 @@ where __name: &'static str, __len: usize, ) -> Result { - Ok(self) + println!("===== tups {:?}", self.ty); + Ok(self.into()) } fn serialize_tuple_variant( @@ -177,17 +223,22 @@ where _variant: &'static str, __len: usize, ) -> Result { - self.0.write_all(&[variant_index as u8])?; - Ok(self) + self.out.write_all(&[variant_index as u8])?; + println!("===== tupv {:?}", self.ty); + Ok(self.into()) } fn serialize_map(self, len: Option) -> Result { - Compact(len.expect("known length") as u64).encode_to(&mut self.0); - Ok(self) + println!("===== m {:?}\n", self.ty); + if !matches!(self.ty(), Some(SerdeType::Struct(_))) { + Compact(len.expect("known length") as u64).encode_to(&mut self.out); + } + Ok(self.into()) } fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { - Ok(self) + println!("===== s {:?}", self.ty); + Ok(self.into()) } fn serialize_struct_variant( @@ -197,12 +248,59 @@ where _variant: &'static str, __len: usize, ) -> Result { - self.0.write_all(&[variant_index as u8])?; - Ok(self) + self.out.write_all(&[variant_index as u8])?; + println!("===== sv {:?}", self.ty); + Ok(self.into()) } } -impl<'a, W> ser::SerializeMap for &'a mut Serializer +impl<'a, W> Serializer +where + W: io::Write, +{ + // A check to run for every serialize fn since any type could be an Option::Some + // if the type info says its an Option assume its Some and extract the inner type + fn maybe_some(&mut self) -> Result<()> { + if let Some(SerdeType::Variant("Option", v)) = self.ty() { + self.ty = v[1].fields().first().map(|f| f.ty().type_info()); + self.out.write_all(&[0x01])?; + } + Ok(()) + } + + fn ty(&'a self) -> Option> { + self.ty.as_ref().map(SerdeType::from) + } +} + +impl<'a, W: 'a> From<&'a mut Serializer> for TypedSerializer<'a, W> { + fn from(ser: &'a mut Serializer) -> Self { + use SerdeType::*; + let t = ser.ty.take(); + match t.as_ref().map(SerdeType::from) { + Some(Struct(fields) | StructTuple(fields)) => { + Self::Collection(ser, fields.iter().map(|f| f.ty().type_info()).collect()) + } + _ => Self::Empty(ser), + } + } +} + +pub enum TypedSerializer<'a, W: 'a> { + Empty(&'a mut Serializer), + Collection(&'a mut Serializer, Vec), +} + +impl<'a, W> TypedSerializer<'a, W> { + fn serializer(&mut self) -> &mut Serializer { + match self { + TypedSerializer::Empty(ser) => *ser, + TypedSerializer::Collection(ser, _) => *ser, + } + } +} + +impl<'a, W> ser::SerializeMap for TypedSerializer<'a, W> where W: io::Write, { @@ -213,14 +311,25 @@ where where T: Serialize, { - key.serialize(&mut **self) + if matches!(self, Self::Collection(_, _)) { + return Ok(()); + } + key.serialize(self.serializer()) } fn serialize_value(&mut self, value: &T) -> Result<()> where T: Serialize, { - value.serialize(&mut **self) + if let Self::Collection(ser, types) = self { + let mut ty = types.remove(0); + // serde_json unwraps newtypes + if let SerdeType::StructNewType(t) = (&ty).into() { + ty = t + } + ser.ty = Some(ty); + } + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -228,7 +337,7 @@ where } } -impl<'a, W> ser::SerializeSeq for &'a mut Serializer +impl<'a, W> ser::SerializeSeq for TypedSerializer<'a, W> where W: io::Write, { @@ -239,7 +348,14 @@ where where T: Serialize, { - value.serialize(&mut **self) + if let Self::Collection(ser, types) = self { + let mut ty = types.remove(0); + if let SerdeType::StructNewType(t) = (&ty).into() { + ty = t + } + ser.ty = Some(ty); + } + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -247,7 +363,7 @@ where } } -impl<'a, W> ser::SerializeStruct for &'a mut Serializer +impl<'a, W> ser::SerializeStruct for TypedSerializer<'a, W> where W: io::Write, { @@ -258,7 +374,7 @@ where where T: Serialize, { - value.serialize(&mut **self) + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -266,7 +382,7 @@ where } } -impl<'a, W> ser::SerializeStructVariant for &'a mut Serializer +impl<'a, W> ser::SerializeStructVariant for TypedSerializer<'a, W> where W: io::Write, { @@ -277,7 +393,7 @@ where where T: Serialize, { - value.serialize(&mut **self) + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -285,7 +401,7 @@ where } } -impl<'a, W> ser::SerializeTuple for &'a mut Serializer +impl<'a, W> ser::SerializeTuple for TypedSerializer<'a, W> where W: io::Write, { @@ -296,7 +412,7 @@ where where T: Serialize, { - value.serialize(&mut **self) + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -304,7 +420,7 @@ where } } -impl<'a, W> ser::SerializeTupleStruct for &'a mut Serializer +impl<'a, W> ser::SerializeTupleStruct for TypedSerializer<'a, W> where W: io::Write, { @@ -315,7 +431,7 @@ where where T: Serialize, { - value.serialize(&mut **self) + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -323,7 +439,7 @@ where } } -impl<'a, W> ser::SerializeTupleVariant for &'a mut Serializer +impl<'a, W> ser::SerializeTupleVariant for TypedSerializer<'a, W> where W: io::Write, { @@ -334,7 +450,7 @@ where where T: Serialize, { - value.serialize(&mut **self) + value.serialize(self.serializer()) } fn end(self) -> Result { @@ -378,6 +494,8 @@ impl From for Error { mod tests { use super::*; use core::mem::size_of; + use scale_info::TypeInfo; + use serde_json::to_value; use std::collections::BTreeMap; #[test] @@ -623,4 +741,75 @@ mod tests { assert_eq!(out, expected); Ok(()) } + + #[test] + fn test_json_simple() -> Result<()> { + #[derive(Debug, Serialize, Encode, TypeInfo)] + struct Foo { + a: Bar, + b: Option, + } + #[derive(Debug, Serialize, Encode, TypeInfo)] + struct Bar(u8); + #[derive(Debug, Serialize, Encode, TypeInfo)] + struct Baz(String, i32); + + let input = Foo { + a: Bar(0xFF), + b: Some(Baz("lol".into(), i32::MIN)), + }; + let mut out = Vec::::new(); + let expected = input.encode(); + + let json_input = to_value(&input).unwrap(); + println!("{:?}\n", input); + println!("{:?}\n", json_input); + to_writer_with_info(&mut out, &json_input, Foo::type_info())?; + + assert_eq!(out, expected); + Ok(()) + } + + #[test] + fn test_json_mix() -> Result<()> { + #[derive(Debug, Serialize, Encode, TypeInfo)] + struct Foo<'a> { + a: Vec, + b: (Bar<'a>, Bar<'a>, Bar<'a>), + } + #[derive(Debug, Serialize, Encode, TypeInfo)] + enum Bar<'a> { + A { thing: &'a str }, + B(Baz), + C(BTreeMap, i64), + } + #[derive(Debug, Serialize, Encode, TypeInfo)] + struct Baz; + + let input = Foo { + a: vec!["hello".into(), "beautiful".into(), "people".into()], + b: ( + Bar::A { thing: "barbarbar" }, + Bar::B(Baz), + Bar::C( + { + let mut h = BTreeMap::new(); + h.insert("key".into(), false); + h + }, + i64::MIN, + ), + ), + }; + let mut out = Vec::::new(); + let expected = input.encode(); + + let json_input = to_value(&input).unwrap(); + println!("{:?}\n", input); + println!("{:?}\n", json_input); + to_writer_with_info(&mut out, &json_input, Foo::type_info())?; + + assert_eq!(out, expected); + Ok(()) + } } From c712aa49f07277d36cef929d368e27be664c8cce Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 17 Sep 2021 16:57:48 +0200 Subject: [PATCH 27/59] Use bytes crate to clean things up --- Cargo.toml | 9 +-- src/lib.rs | 9 ++- src/value.rs | 202 +++++++++++++++++++++++---------------------------- 3 files changed, 102 insertions(+), 118 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 68e505e..743b0e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,16 +7,15 @@ edition = "2018" repository = "https://github.com/valibre-org/scales" [dependencies] -byteorder = { version = "1.4.3", default-features = false } -codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, optional = true } -core2 = { version = "0.3.1", default-features = false, optional = true } +bytes = { version = "1.0.1", default-features = false } scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } +codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, optional = true } [features] default = ["std", "serializer"] -std = ["scale-info/std", "core2/std"] -serializer = ["core2", "codec"] +std = ["scale-info/std", "bytes/std"] +serializer = ["codec"] [dev-dependencies] anyhow = "1.0.42" diff --git a/src/lib.rs b/src/lib.rs index 2c0784c..60a5254 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -72,7 +72,14 @@ impl<'a> From<&'a Type> for SerdeType<'a> { } } Variant(v) => Self::Variant(name, v.variants()), - Sequence(s) => Self::Sequence(s.type_param().type_info()), + Sequence(s) => { + let ty = s.type_param().type_info(); + if ty.path().segments() != &["u8"] { + Self::Sequence(ty) + } else { + Self::Bytes + } + } Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param(), a.len())), Tuple(t) => Self::Tuple(TupleOrArray::Tuple(t.fields())), Primitive(p) => match p { diff --git a/src/value.rs b/src/value.rs index c2d4d6a..82e634e 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,6 +1,5 @@ use crate::{EnumVariant, SerdeType}; -use byteorder::{ByteOrder, LE}; -use core::cell::Cell; +use bytes::{Buf, Bytes}; use core::convert::TryInto; use core::str; use scale_info::{prelude::*, Type, TypeDef, TypeDefPrimitive as Primitive}; @@ -14,143 +13,145 @@ use serde::Serialize; /// directly with the help of a type registry and without using an /// intermediate representation that requires allocating data. #[derive(Debug)] -pub struct Value<'a> { - data: &'a [u8], +pub struct Value { + data: Bytes, ty: Type, - idx: Cell, } -impl<'a> Value<'a> { - pub fn new(data: &'a [u8], ty: Type) -> Self { +impl Value { + pub fn new(data: impl Into, ty: Type) -> Self { Value { - data, + data: data.into(), ty, - idx: 0.into(), } } + + fn chunk(mut data: impl Buf, ty: Type) -> Self { + let size = type_len(&ty, data.chunk()); + Value::new(data.copy_to_bytes(size), ty) + } + + #[inline] + fn ty_name(&self) -> &'static str { + self.ty.path().segments().last().copied().unwrap_or("") + } } -impl<'a> Serialize for Value<'a> { +impl Serialize for Value { fn serialize(&self, ser: S) -> Result where S: serde::Serializer, { - let name = self.ty_name(); - let data = self.data_left(); - let ty = SerdeType::from(&self.ty); + let mut data = self.data.clone(); use SerdeType::*; - match ty { - Bool => ser.serialize_bool(data[0] != 0), - U8 => ser.serialize_u8(data[0]), - U16 => ser.serialize_u16(LE::read_u16(data)), - U32 => ser.serialize_u32(LE::read_u32(data)), - U64 => ser.serialize_u64(LE::read_u64(data)), - U128 => ser.serialize_u128(LE::read_u128(data)), - I8 => ser.serialize_i8(i8::from_le_bytes([data[0]])), - I16 => ser.serialize_i16(LE::read_i16(data)), - I32 => ser.serialize_i32(LE::read_i32(data)), - I64 => ser.serialize_i64(LE::read_i64(data)), - I128 => ser.serialize_i128(LE::read_i128(data)), - Bytes => unimplemented!(), - Char => { - let n = LE::read_u32(data); - ser.serialize_char(char::from_u32(n).unwrap()) - } + match (&self.ty).into() { + Bool => ser.serialize_bool(data.get_u8() != 0), + U8 => ser.serialize_u8(data.get_u8()), + U16 => ser.serialize_u16(data.get_u16_le()), + U32 => ser.serialize_u32(data.get_u32_le()), + U64 => ser.serialize_u64(data.get_u64_le()), + U128 => ser.serialize_u128(data.get_u128_le()), + I8 => ser.serialize_i8(data.get_i8()), + I16 => ser.serialize_i16(data.get_i16_le()), + I32 => ser.serialize_i32(data.get_i32_le()), + I64 => ser.serialize_i64(data.get_i64_le()), + I128 => ser.serialize_i128(data.get_i128_le()), + Bytes => ser.serialize_bytes(data.chunk()), + Char => ser.serialize_char(char::from_u32(data.get_u32_le()).unwrap()), Str => { - let (_, s) = sequence_len(data); - self.advance_idx(s); - ser.serialize_str(str::from_utf8(self.data_left()).unwrap()) + let (_, s) = sequence_len(data.chunk()); + data.advance(s); + ser.serialize_str(str::from_utf8(data.chunk()).unwrap()) } Sequence(ty) => { - let (len, p_size) = sequence_len(self.data_left()); - self.advance_idx(p_size); + let (len, p_size) = sequence_len(data.chunk()); + data.advance(p_size); let mut seq = ser.serialize_seq(Some(len))?; for _ in 0..len { - seq.serialize_element(&self.sub_value(ty.clone()))?; + seq.serialize_element(&Value::chunk(&mut data, ty.clone()))?; } seq.end() } Map(ty_k, ty_v) => { - let (len, p_size) = sequence_len(self.data_left()); - self.advance_idx(p_size); + let (len, p_size) = sequence_len(data.chunk()); + data.advance(p_size); let mut state = ser.serialize_map(Some(len))?; for _ in 0..len { - let key = &self.sub_value(ty_k.clone()); - let val = &self.sub_value(ty_v.clone()); - state.serialize_entry(key, val)?; + let key = Value::chunk(&mut data, ty_k.clone()); + let val = Value::chunk(&mut data, ty_v.clone()); + state.serialize_entry(&key, &val)?; } state.end() } Tuple(t) => { let mut state = ser.serialize_tuple(t.len())?; for i in 0..t.len() { - state.serialize_element(&self.sub_value(t.type_info(i)))?; + state.serialize_element(&Value::chunk(&mut data, t.type_info(i)))?; } state.end() } Struct(fields) => { - let mut state = ser.serialize_struct(name, fields.len())?; + let mut state = ser.serialize_struct(self.ty_name(), fields.len())?; for f in fields { - state - .serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; + state.serialize_field( + f.name().unwrap(), + &Value::chunk(&mut data, f.ty().type_info()), + )?; } state.end() } - StructUnit => ser.serialize_unit_struct(name), - StructNewType(ty) => ser.serialize_newtype_struct(name, &self.sub_value(ty)), + StructUnit => ser.serialize_unit_struct(self.ty_name()), + StructNewType(ty) => { + ser.serialize_newtype_struct(self.ty_name(), &Value::chunk(&mut data, ty)) + } StructTuple(fields) => { - let mut state = ser.serialize_tuple_struct(name, fields.len())?; + let mut state = ser.serialize_tuple_struct(self.ty_name(), fields.len())?; for f in fields { - state.serialize_field(&self.sub_value(f.ty().type_info()))?; + state.serialize_field(&Value::chunk(&mut data, f.ty().type_info()))?; } state.end() } - Variant(_, _) => match ty.pick_variant(data[0]) { + ty @ Variant(_, _) => match ty.pick_variant(data.get_u8()) { EnumVariant::OptionNone => ser.serialize_none(), - EnumVariant::OptionSome(ty) => { - self.advance_idx(1); - ser.serialize_some(&self.sub_value(ty)) - } + EnumVariant::OptionSome(ty) => ser.serialize_some(&Value::chunk(&mut data, ty)), EnumVariant::Unit(v) => { - ser.serialize_unit_variant(name, v.index().into(), v.name()) + ser.serialize_unit_variant(self.ty_name(), v.index().into(), v.name()) } EnumVariant::NewType(v) => { - self.advance_idx(1); let ty = v.fields().first().unwrap().ty().type_info(); ser.serialize_newtype_variant( - name, + self.ty_name(), v.index().into(), v.name(), - &self.sub_value(ty), + &Value::chunk(&mut data, ty), ) } EnumVariant::Tuple(v) => { - self.advance_idx(1); let mut s = ser.serialize_tuple_variant( - name, + self.ty_name(), v.index().into(), v.name(), v.fields().len(), )?; for f in v.fields() { - s.serialize_field(&self.sub_value(f.ty().type_info()))?; + s.serialize_field(&Value::chunk(&mut data, f.ty().type_info()))?; } s.end() } EnumVariant::Struct(v) => { - self.advance_idx(1); let mut s = ser.serialize_struct_variant( - name, + self.ty_name(), v.index().into(), v.name(), v.fields().len(), )?; for f in v.fields() { - s.serialize_field(f.name().unwrap(), &self.sub_value(f.ty().type_info()))?; + let ty = f.ty().type_info(); + s.serialize_field(f.name().unwrap(), &Value::chunk(&mut data, ty))?; } s.end() } @@ -159,32 +160,7 @@ impl<'a> Serialize for Value<'a> { } } -impl<'a> Value<'a> { - fn data_left(&self) -> &[u8] { - &self.data[self.idx.get()..] - } - - fn sub_value(&self, ty: Type) -> Value { - let size = ty_data_size(&ty, self.data_left()); - Value::new(self.extract_data(size), ty) - } - - fn extract_data(&self, end: usize) -> &[u8] { - let (data, _) = &self.data_left().split_at(end); - self.advance_idx(end); - data - } - - fn advance_idx(&self, n: usize) { - self.idx.set(self.idx.get() + n) - } - - fn ty_name(&self) -> &'static str { - self.ty.path().segments().last().copied().unwrap_or("") - } -} - -fn ty_data_size(ty: &Type, data: &[u8]) -> usize { +fn type_len(ty: &Type, data: &[u8]) -> usize { match ty.type_def() { TypeDef::Primitive(p) => match p { Primitive::U8 => mem::size_of::(), @@ -208,7 +184,7 @@ fn ty_data_size(ty: &Type, data: &[u8]) -> usize { TypeDef::Composite(c) => c .fields() .iter() - .fold(0, |c, f| c + ty_data_size(&f.ty().type_info(), &data[c..])), + .fold(0, |c, f| c + type_len(&f.ty().type_info(), &data[c..])), TypeDef::Variant(e) => { let var = e .variants() @@ -221,13 +197,13 @@ fn ty_data_size(ty: &Type, data: &[u8]) -> usize { } else { var.fields() .iter() - .fold(1, |c, f| c + ty_data_size(&f.ty().type_info(), &data[c..])) + .fold(1, |c, f| c + type_len(&f.ty().type_info(), &data[c..])) } } TypeDef::Sequence(s) => { let (len, prefix_size) = sequence_len(data); let ty = s.type_param().type_info(); - (0..len).fold(prefix_size, |c, _| c + ty_data_size(&ty, &data[c..])) + (0..len).fold(prefix_size, |c, _| c + type_len(&ty, &data[c..])) } TypeDef::Array(a) => a.len().try_into().unwrap(), TypeDef::Tuple(t) => t.fields().len(), @@ -283,7 +259,7 @@ mod tests { let extract_value = u8::MAX; let data = extract_value.encode(); let info = u8::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -293,7 +269,7 @@ mod tests { let extract_value = u16::MAX; let data = extract_value.encode(); let info = u16::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -303,7 +279,7 @@ mod tests { let extract_value = u32::MAX; let data = extract_value.encode(); let info = u32::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -313,7 +289,7 @@ mod tests { let extract_value = u64::MAX; let data = extract_value.encode(); let info = u64::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -323,7 +299,7 @@ mod tests { let extract_value = i16::MAX; let data = extract_value.encode(); let info = i16::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -333,7 +309,7 @@ mod tests { let extract_value = i32::MAX; let data = extract_value.encode(); let info = i32::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -343,7 +319,7 @@ mod tests { let extract_value = i64::MAX; let data = extract_value.encode(); let info = i64::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -353,7 +329,7 @@ mod tests { let extract_value = true; let data = extract_value.encode(); let info = bool::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -364,7 +340,7 @@ mod tests { // let extract_value = '⚖'; // let data = extract_value.encode(); // let info = char::type_info(); - // let val = Value::new(&data, info); + // let val = Value::new(data, info); // assert_eq!(to_value(val)?, to_value(extract_value)?); // Ok(()) // } @@ -374,7 +350,7 @@ mod tests { let extract_value: Vec = [2u8, u8::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -384,7 +360,7 @@ mod tests { let extract_value: Vec = [2u16, u16::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -394,7 +370,7 @@ mod tests { let extract_value: Vec = [2u32, u32::MAX].into(); let data = extract_value.encode(); let info = Vec::::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -408,7 +384,7 @@ mod tests { ); let data = extract_value.encode(); let info = <(i64, Vec, bool)>::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) } @@ -426,7 +402,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) @@ -445,7 +421,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) @@ -464,7 +440,7 @@ mod tests { }; let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) @@ -481,7 +457,7 @@ mod tests { let data = value.encode(); let info = BTreeMap::::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(value)?); Ok(()) @@ -500,14 +476,16 @@ mod tests { struct Foo { bar: Vec, baz: Option, + lol: &'static [u8], } let expected = Foo { bar: [Bar::That(i16::MAX), Bar::This].into(), baz: Some(Baz("aliquam malesuada bibendum arcu vitae".into())), + lol: b"\0xFFsome stuff\0x00", }; let data = expected.encode(); let info = Foo::type_info(); - let out = Value::new(&data, info); + let out = Value::new(data, info); assert_eq!(to_value(out)?, to_value(expected)?); Ok(()) @@ -535,7 +513,7 @@ mod tests { ); let data = extract_value.encode(); let info = Foo::type_info(); - let val = Value::new(&data, info); + let val = Value::new(data, info); assert_eq!(to_value(val)?, to_value(extract_value)?); Ok(()) From ce5abe163a926a2c1daac8097fee79084521b519 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 17 Sep 2021 20:57:23 +0200 Subject: [PATCH 28/59] Remove core2 and scale-codec dependecy --- Cargo.toml | 5 +- src/lib.rs | 4 +- src/serializer.rs | 221 ++++++++++++++++++++++++++++++---------------- 3 files changed, 150 insertions(+), 80 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 743b0e5..0b32a16 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,12 +10,11 @@ repository = "https://github.com/valibre-org/scales" bytes = { version = "1.0.1", default-features = false } scale-info = { version = "0.10.0", default-features = false } serde = { version = "1.0.126", default-features = false } -codec = { version = "2.2.0", package = "parity-scale-codec", default-features = false, optional = true } [features] -default = ["std", "serializer"] +default = ["std", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] -serializer = ["codec"] +experimental-serializer = [] [dev-dependencies] anyhow = "1.0.42" diff --git a/src/lib.rs b/src/lib.rs index 60a5254..5223545 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,11 +4,11 @@ ///! Dynamic SCALE Serialization using `scale-info` type information. ///! -#[cfg(feature = "serializer")] +#[cfg(feature = "experimental-serializer")] mod serializer; mod value; -#[cfg(feature = "serializer")] +#[cfg(feature = "experimental-serializer")] pub use serializer::{to_writer, Serializer}; pub use value::Value; diff --git a/src/serializer.rs b/src/serializer.rs index 2b00cf5..3beabb5 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,28 +1,27 @@ -use codec::{Compact, Encode}; +use bytes::BufMut; use core::fmt; -use core2::io; use scale_info::{Type, TypeInfo}; use serde::{ser, Serialize}; -use crate::SerdeType; +use crate::{SerdeType, TupleOrArray}; type Result = core::result::Result; #[derive(TypeInfo)] struct Noop; -pub fn to_writer(writer: W, value: &T) -> Result<()> +pub fn to_writer(bytes: B, value: &T) -> Result<()> where T: Serialize, - W: io::Write, + B: BufMut, { - to_writer_with_info(writer, value, Noop::type_info()) + to_writer_with_info(bytes, value, Noop::type_info()) } -pub fn to_writer_with_info(writer: W, value: &T, info: Type) -> Result<()> +pub fn to_writer_with_info(writer: B, value: &T, info: Type) -> Result<()> where T: Serialize, - W: io::Write, + B: BufMut, { let info = if info.path().segments().eq(&["scales", "serializer", "Noop"]) { None @@ -34,85 +33,99 @@ where Ok(()) } -pub struct Serializer { - out: W, +pub struct Serializer { + out: B, ty: Option, } -impl Serializer +impl Serializer where - W: io::Write, + B: BufMut, { - pub fn new(out: W, ty: Option) -> Self { + pub fn new(out: B, ty: Option) -> Self { Serializer { out, ty } } } -impl<'a, W> ser::Serializer for &'a mut Serializer +impl<'a, B> ser::Serializer for &'a mut Serializer where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; - type SerializeSeq = TypedSerializer<'a, W>; - type SerializeTuple = TypedSerializer<'a, W>; - type SerializeTupleStruct = TypedSerializer<'a, W>; - type SerializeTupleVariant = TypedSerializer<'a, W>; - type SerializeMap = TypedSerializer<'a, W>; - type SerializeStruct = TypedSerializer<'a, W>; - type SerializeStructVariant = TypedSerializer<'a, W>; + type SerializeSeq = TypedSerializer<'a, B>; + type SerializeTuple = TypedSerializer<'a, B>; + type SerializeTupleStruct = TypedSerializer<'a, B>; + type SerializeTupleVariant = TypedSerializer<'a, B>; + type SerializeMap = TypedSerializer<'a, B>; + type SerializeStruct = TypedSerializer<'a, B>; + type SerializeStructVariant = TypedSerializer<'a, B>; fn serialize_bool(self, v: bool) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + self.out.put_u8(v.into()); Ok(()) } fn serialize_i8(self, v: i8) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + self.out.put_i8(v); Ok(()) } fn serialize_i16(self, v: i16) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + self.out.put_i16_le(v); Ok(()) } fn serialize_i32(self, v: i32) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + self.out.put_i32_le(v); Ok(()) } fn serialize_i64(self, v: i64) -> Result { match self.ty() { - Some(SerdeType::I8) => (v as i8).encode_to(&mut self.out), - Some(SerdeType::I16) => (v as i16).encode_to(&mut self.out), - Some(SerdeType::I32) => (v as i32).encode_to(&mut self.out), - _ => v.encode_to(&mut self.out), + Some(SerdeType::I8) => self.serialize_i8(v as i8)?, + Some(SerdeType::I16) => self.serialize_i16(v as i16)?, + Some(SerdeType::I32) => self.serialize_i32(v as i32)?, + _ => { + self.maybe_some()?; + self.out.put_i64_le(v) + } } Ok(()) } fn serialize_u8(self, v: u8) -> Result { - self.out.write_all(&[v]).map_err(Error::from) + self.maybe_some()?; + self.out.put_u8(v); + Ok(()) } fn serialize_u16(self, v: u16) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + self.out.put_u16_le(v); Ok(()) } fn serialize_u32(self, v: u32) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + self.out.put_u32_le(v); Ok(()) } fn serialize_u64(self, v: u64) -> Result { match self.ty() { - Some(SerdeType::U8) => (v as u8).encode_to(&mut self.out), - Some(SerdeType::U16) => (v as u16).encode_to(&mut self.out), - Some(SerdeType::U32) => (v as u32).encode_to(&mut self.out), - _ => v.encode_to(&mut self.out), + Some(SerdeType::U8) => self.serialize_u8(v as u8)?, + Some(SerdeType::U16) => self.serialize_u16(v as u16)?, + Some(SerdeType::U32) => self.serialize_u32(v as u32)?, + _ => { + self.maybe_some()?; + self.out.put_u64_le(v); + } } Ok(()) } @@ -130,32 +143,39 @@ where } fn serialize_str(self, v: &str) -> Result { - v.encode_to(&mut self.out); + self.maybe_some()?; + compact_number(v.len(), &mut self.out); + self.out.put(v.as_bytes()); Ok(()) } fn serialize_bytes(self, v: &[u8]) -> Result { - self.out.write_all(v).map_err(Error::from) + self.maybe_some()?; + self.out.put(v); + Ok(()) } fn serialize_none(self) -> Result { - self.out.write_all(&[0x00]).map_err(Error::from) + self.out.put_u8(0x00); + Ok(()) } fn serialize_some(self, value: &T) -> Result where T: Serialize, { - self.out.write_all(&[0x01])?; + self.out.put_u8(0x01); value.serialize(self) } fn serialize_unit(self) -> Result { + self.maybe_some()?; println!("===== u {:?}\n", self.ty); Ok(()) } fn serialize_unit_struct(self, _name: &'static str) -> Result { + self.maybe_some()?; println!("===== us {:?}\n", self.ty); Ok(()) } @@ -166,6 +186,7 @@ where variant_index: u32, _variant: &'static str, ) -> Result { + self.maybe_some()?; println!("===== uv {:?}\n", self.ty); (variant_index as u8).serialize(self) } @@ -174,6 +195,7 @@ where where T: Serialize, { + self.maybe_some()?; println!("===== ns {:?}\n", self.ty); value.serialize(self) } @@ -188,8 +210,9 @@ where where T: Serialize, { + self.maybe_some()?; println!("===== nv {:?}\n", self.ty); - self.out.write_all(&[variant_index as u8])?; + self.out.put_u8(variant_index as u8); value.serialize(self) } @@ -197,12 +220,13 @@ where self.maybe_some()?; println!("===== seq {:?}\n", self.ty); if !matches!(self.ty(), Some(SerdeType::StructTuple(_))) { - Compact(len.expect("known length") as u64).encode_to(&mut self.out); + compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) } fn serialize_tuple(self, _len: usize) -> Result { + self.maybe_some()?; println!("===== tup {:?}\n", self.ty); Ok(self.into()) } @@ -212,6 +236,7 @@ where __name: &'static str, __len: usize, ) -> Result { + self.maybe_some()?; println!("===== tups {:?}", self.ty); Ok(self.into()) } @@ -223,20 +248,26 @@ where _variant: &'static str, __len: usize, ) -> Result { - self.out.write_all(&[variant_index as u8])?; + self.maybe_some()?; + self.out.put_u8(variant_index as u8); println!("===== tupv {:?}", self.ty); Ok(self.into()) } fn serialize_map(self, len: Option) -> Result { + self.maybe_some()?; println!("===== m {:?}\n", self.ty); + if matches!(self.ty(), Some(SerdeType::Variant(_, _))) { + println!("map as variant: {:?}\n", len); + } if !matches!(self.ty(), Some(SerdeType::Struct(_))) { - Compact(len.expect("known length") as u64).encode_to(&mut self.out); + compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) } fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + self.maybe_some()?; println!("===== s {:?}", self.ty); Ok(self.into()) } @@ -248,22 +279,23 @@ where _variant: &'static str, __len: usize, ) -> Result { - self.out.write_all(&[variant_index as u8])?; + self.maybe_some()?; + self.out.put_u8(variant_index as u8); println!("===== sv {:?}", self.ty); Ok(self.into()) } } -impl<'a, W> Serializer +impl<'a, B> Serializer where - W: io::Write, + B: BufMut, { // A check to run for every serialize fn since any type could be an Option::Some // if the type info says its an Option assume its Some and extract the inner type fn maybe_some(&mut self) -> Result<()> { if let Some(SerdeType::Variant("Option", v)) = self.ty() { self.ty = v[1].fields().first().map(|f| f.ty().type_info()); - self.out.write_all(&[0x01])?; + self.out.put_u8(0x01); } Ok(()) } @@ -273,14 +305,22 @@ where } } -impl<'a, W: 'a> From<&'a mut Serializer> for TypedSerializer<'a, W> { - fn from(ser: &'a mut Serializer) -> Self { +impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { + fn from(ser: &'a mut Serializer) -> Self { use SerdeType::*; + let t = ser.ty.take(); match t.as_ref().map(SerdeType::from) { Some(Struct(fields) | StructTuple(fields)) => { Self::Collection(ser, fields.iter().map(|f| f.ty().type_info()).collect()) } + Some(Tuple(TupleOrArray::Array(ty, n))) => { + Self::Collection(ser, (0..n).map(|_| ty.type_info()).collect()) + } + Some(Tuple(TupleOrArray::Tuple(fields))) => { + Self::Collection(ser, fields.iter().map(|f| f.type_info()).collect()) + } + Some(Variant(_, variants)) => Self::Variant(ser, variants.into()), _ => Self::Empty(ser), } } @@ -289,20 +329,20 @@ impl<'a, W: 'a> From<&'a mut Serializer> for TypedSerializer<'a, W> { pub enum TypedSerializer<'a, W: 'a> { Empty(&'a mut Serializer), Collection(&'a mut Serializer, Vec), + Variant(&'a mut Serializer, Vec), } -impl<'a, W> TypedSerializer<'a, W> { - fn serializer(&mut self) -> &mut Serializer { +impl<'a, B> TypedSerializer<'a, B> { + fn serializer(&mut self) -> &mut Serializer { match self { - TypedSerializer::Empty(ser) => *ser, - TypedSerializer::Collection(ser, _) => *ser, + Self::Empty(ser) | Self::Collection(ser, _) | Self::Variant(ser, _) => ser, } } } -impl<'a, W> ser::SerializeMap for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeMap for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -314,7 +354,20 @@ where if matches!(self, Self::Collection(_, _)) { return Ok(()); } - key.serialize(self.serializer()) + match self { + TypedSerializer::Empty(_) => key.serialize(self.serializer()), + TypedSerializer::Collection(_, _) => Ok(()), + TypedSerializer::Variant(_, variants) => { + let key_data = { + let mut s = Serializer::new(vec![], None); + key.serialize(&mut s)?; + s.out + }; + + println!("=========> {:?}: {:?}\n", key_data, variants); + Ok(()) + } + } } fn serialize_value(&mut self, value: &T) -> Result<()> @@ -337,9 +390,9 @@ where } } -impl<'a, W> ser::SerializeSeq for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeSeq for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -363,9 +416,9 @@ where } } -impl<'a, W> ser::SerializeStruct for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeStruct for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -382,9 +435,9 @@ where } } -impl<'a, W> ser::SerializeStructVariant for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeStructVariant for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -401,9 +454,9 @@ where } } -impl<'a, W> ser::SerializeTuple for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeTuple for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -420,9 +473,9 @@ where } } -impl<'a, W> ser::SerializeTupleStruct for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeTupleStruct for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -439,9 +492,9 @@ where } } -impl<'a, W> ser::SerializeTupleVariant for TypedSerializer<'a, W> +impl<'a, B> ser::SerializeTupleVariant for TypedSerializer<'a, B> where - W: io::Write, + B: BufMut, { type Ok = (); type Error = Error; @@ -461,14 +514,12 @@ where #[derive(Debug)] pub enum Error { Ser(String), - Io(io::Error), } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Error::Ser(msg) => write!(f, "{}", msg), - Error::Io(e) => write!(f, "{}", e), } } } @@ -484,15 +535,35 @@ impl ser::Error for Error { } } -impl From for Error { - fn from(e: io::Error) -> Self { - Error::Io(e) +fn compact_number(n: usize, mut dest: impl BufMut) { + match n { + 0..=0b0011_1111 => dest.put_u8((n as u8) << 2), + 0..=0b0011_1111_1111_1111 => dest.put_u16_le(((n as u16) << 2) | 0b01), + 0..=0b0011_1111_1111_1111_1111_1111_1111_1111 => dest.put_u32_le(((n as u32) << 2) | 0b10), + _ => { + let bytes_needed = 8 - n.leading_zeros() / 8; + assert!( + bytes_needed >= 4, + "Previous match arm matches anyting less than 2^30; qed" + ); + dest.put_u8(0b11 + ((bytes_needed - 4) << 2) as u8); + let mut v = n; + for _ in 0..bytes_needed { + dest.put_u8(v as u8); + v >>= 8; + } + assert_eq!( + v, 0, + "shifted sufficient bits right to lead only leading zeros; qed" + ) + } } } #[cfg(test)] mod tests { use super::*; + use codec::Encode; use core::mem::size_of; use scale_info::TypeInfo; use serde_json::to_value; From 89caf88440b02ccb866db1948e55dd7dfe250d74 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 20 Sep 2021 19:51:05 +0200 Subject: [PATCH 29/59] SerdeType uses owned types --- src/lib.rs | 126 +++++++++++++++++++++---------------- src/serializer.rs | 154 ++++++++++++++++++++++++++-------------------- src/value.rs | 93 ++++++++++++++-------------- 3 files changed, 208 insertions(+), 165 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5223545..5de8ea8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,16 +9,22 @@ mod serializer; mod value; #[cfg(feature = "experimental-serializer")] -pub use serializer::{to_writer, Serializer}; +pub use serializer::{to_bytes, Serializer}; pub use value::Value; use scale_info::{Field, MetaType, Type, Variant}; +macro_rules! is_tuple { + ($it:ident) => { + $it.fields().first().and_then(Field::name).is_none() + }; +} + /// A convenient representation of the scale-info types to a format /// that matches serde model more closely #[rustfmt::skip] #[derive(Debug)] -enum SerdeType<'a> { +enum SerdeType { Bool, U8, U16, U32, U64, U128, I8, I16, I32, I64, I128, @@ -27,13 +33,13 @@ enum SerdeType<'a> { Str, Sequence(Type), Map(Type, Type), - Tuple(TupleOrArray<'a>), - Struct(&'a [Field]), StructUnit, StructNewType(Type), StructTuple(&'a [Field]), - Variant(&'a str, &'a [Variant]), + Tuple(TupleOrArray), + Struct(Vec), StructUnit, StructNewType(Type), StructTuple(Vec), + Variant(String, Vec, Option), } -impl<'a> From<&'a Type> for SerdeType<'a> { - fn from(ty: &'a Type) -> Self { +impl From for SerdeType { + fn from(ty: Type) -> Self { use scale_info::{TypeDef, TypeDef::*, TypeDefComposite, TypeDefPrimitive}; let name = ty.path().segments().last().copied().unwrap_or(""); @@ -65,13 +71,13 @@ impl<'a> From<&'a Type> for SerdeType<'a> { Self::Map(k, v) } else if fields.len() == 1 { Self::StructNewType(fields.first().unwrap().ty().type_info()) - } else if is_tuple(fields) { - Self::StructTuple(fields) + } else if is_tuple!(c) { + Self::StructTuple(fields.into()) } else { - Self::Struct(fields) + Self::Struct(fields.into()) } } - Variant(v) => Self::Variant(name, v.variants()), + Variant(v) => Self::Variant(name.into(), v.variants().into(), None), Sequence(s) => { let ty = s.type_param().type_info(); if ty.path().segments() != &["u8"] { @@ -80,8 +86,10 @@ impl<'a> From<&'a Type> for SerdeType<'a> { Self::Bytes } } - Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param(), a.len())), - Tuple(t) => Self::Tuple(TupleOrArray::Tuple(t.fields())), + Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param().type_info(), a.len())), + Tuple(t) => Self::Tuple(TupleOrArray::Tuple( + t.fields().iter().map(MetaType::type_info).collect(), + )), Primitive(p) => match p { TypeDefPrimitive::U8 => Self::U8, TypeDefPrimitive::U16 => Self::U16, @@ -105,57 +113,76 @@ impl<'a> From<&'a Type> for SerdeType<'a> { } } -impl<'a> SerdeType<'a> { - fn pick_variant(self, index: u8) -> EnumVariant<'a> { +impl SerdeType { + fn pick(&self, index: u8) -> Self { match self { - SerdeType::Variant(name, variants) => { - let variant = variants - .iter() - .find(|v| v.index() == index) - .expect("variant"); + SerdeType::Variant(name, variant, Some(_)) => { + Self::Variant(name.to_string(), variant.to_vec(), Some(index)) + } + SerdeType::Variant(name, variants, None) => { + let v = variants.iter().find(|v| v.index() == index).unwrap(); + Self::Variant(name.clone(), vec![v.clone()], Some(index)) + } + _ => panic!("Only for enum variants"), + } + } +} + +#[derive(Debug)] +enum EnumVariant<'a> { + OptionNone, + OptionSome(Type), + Unit(u8, &'a str), + NewType(u8, &'a str, Type), + Tuple(u8, &'a str, Vec), + Struct(u8, &'a str, Vec<(&'a str, Type)>), +} + +impl<'a> From<&SerdeType> for EnumVariant<'a> { + fn from(ty: &SerdeType) -> Self { + match ty { + SerdeType::Variant(name, variants, Some(idx)) => { + let variant = variants.first().expect("single variant"); let fields = variant.fields(); + let vname = *variant.name(); + if fields.is_empty() { - if name == "Option" && variant.name() == &"None" { - EnumVariant::OptionNone.into() + if name == "Option" && vname == "None" { + Self::OptionNone } else { - EnumVariant::Unit(variant).into() + Self::Unit(*idx, vname) } - } else if is_tuple(fields) { + } else if is_tuple!(variant) { if fields.len() == 1 { - let ty = fields.first().unwrap().ty().type_info(); + let ty = fields.first().map(|f| f.ty().type_info()).unwrap(); return if name == "Option" && variant.name() == &"Some" { - EnumVariant::OptionSome(ty).into() + Self::OptionSome(ty) } else { - EnumVariant::NewType(variant).into() + Self::NewType(*idx, vname, ty) }; } else { - EnumVariant::Tuple(variant).into() + let fields = fields.iter().map(|f| f.ty().type_info()).collect(); + Self::Tuple(*idx, vname, fields) } } else { - EnumVariant::Struct(variant).into() + let fields = fields + .iter() + .map(|f| (*f.name().unwrap(), f.ty().type_info())) + .collect(); + Self::Struct(*idx, vname, fields) } } - _ => unreachable!(), + _ => panic!("Only for enum variants"), } } } #[derive(Debug)] -enum EnumVariant<'a> { - OptionNone, - OptionSome(Type), - Unit(&'a Variant), - NewType(&'a Variant), - Tuple(&'a Variant), - Struct(&'a Variant), +enum TupleOrArray { + Array(Type, u32), + Tuple(Vec), } - -#[derive(Debug)] -enum TupleOrArray<'a> { - Array(&'a MetaType, u32), - Tuple(&'a [MetaType]), -} -impl<'a> TupleOrArray<'a> { +impl TupleOrArray { fn len(&self) -> usize { match self { Self::Array(_, len) => *len as usize, @@ -163,15 +190,10 @@ impl<'a> TupleOrArray<'a> { } } - fn type_info(&self, i: usize) -> Type { + fn type_info(&self, i: usize) -> &Type { match self { - Self::Array(ty, _) => ty.type_info(), - Self::Tuple(fields) => fields[i].type_info(), + Self::Array(ty, _) => ty, + Self::Tuple(fields) => &fields[i], } } } - -#[inline] -fn is_tuple(fields: &[Field]) -> bool { - fields.first().and_then(Field::name).is_none() -} diff --git a/src/serializer.rs b/src/serializer.rs index 3beabb5..6d47b7f 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -10,24 +10,19 @@ type Result = core::result::Result; #[derive(TypeInfo)] struct Noop; -pub fn to_writer(bytes: B, value: &T) -> Result<()> +pub fn to_bytes(bytes: B, value: &T) -> Result<()> where T: Serialize, B: BufMut, { - to_writer_with_info(bytes, value, Noop::type_info()) + to_bytes_with_info(bytes, value, None) } -pub fn to_writer_with_info(writer: B, value: &T, info: Type) -> Result<()> +pub fn to_bytes_with_info(writer: B, value: &T, info: impl Into>) -> Result<()> where T: Serialize, B: BufMut, { - let info = if info.path().segments().eq(&["scales", "serializer", "Noop"]) { - None - } else { - Some(info) - }; let mut serializer = Serializer::new(writer, info); value.serialize(&mut serializer)?; Ok(()) @@ -35,15 +30,18 @@ where pub struct Serializer { out: B, - ty: Option, + ty: Option, } impl Serializer where B: BufMut, { - pub fn new(out: B, ty: Option) -> Self { - Serializer { out, ty } + pub fn new(out: B, ty: impl Into>) -> Self { + Serializer { + out, + ty: ty.into().map(SerdeType::from), + } } } @@ -87,7 +85,7 @@ where } fn serialize_i64(self, v: i64) -> Result { - match self.ty() { + match self.ty { Some(SerdeType::I8) => self.serialize_i8(v as i8)?, Some(SerdeType::I16) => self.serialize_i16(v as i16)?, Some(SerdeType::I32) => self.serialize_i32(v as i32)?, @@ -118,7 +116,7 @@ where } fn serialize_u64(self, v: u64) -> Result { - match self.ty() { + match self.ty { Some(SerdeType::U8) => self.serialize_u8(v as u8)?, Some(SerdeType::U16) => self.serialize_u16(v as u16)?, Some(SerdeType::U32) => self.serialize_u32(v as u32)?, @@ -219,7 +217,7 @@ where fn serialize_seq(self, len: Option) -> Result { self.maybe_some()?; println!("===== seq {:?}\n", self.ty); - if !matches!(self.ty(), Some(SerdeType::StructTuple(_))) { + if !matches!(self.ty, Some(SerdeType::StructTuple(_))) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -257,10 +255,11 @@ where fn serialize_map(self, len: Option) -> Result { self.maybe_some()?; println!("===== m {:?}\n", self.ty); - if matches!(self.ty(), Some(SerdeType::Variant(_, _))) { + if matches!(self.ty, Some(SerdeType::Variant(_, _, _))) { + // TODO! println!("map as variant: {:?}\n", len); } - if !matches!(self.ty(), Some(SerdeType::Struct(_))) { + if !matches!(self.ty, Some(SerdeType::Struct(_))) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -293,16 +292,15 @@ where // A check to run for every serialize fn since any type could be an Option::Some // if the type info says its an Option assume its Some and extract the inner type fn maybe_some(&mut self) -> Result<()> { - if let Some(SerdeType::Variant("Option", v)) = self.ty() { - self.ty = v[1].fields().first().map(|f| f.ty().type_info()); - self.out.put_u8(0x01); + match &self.ty { + Some(SerdeType::Variant(ref name, v, _)) if name == "Option" => { + self.ty = v[1].fields().first().map(|f| f.ty().type_info().into()); + self.out.put_u8(0x01); + } + _ => (), } Ok(()) } - - fn ty(&'a self) -> Option> { - self.ty.as_ref().map(SerdeType::from) - } } impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { @@ -310,32 +308,42 @@ impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { use SerdeType::*; let t = ser.ty.take(); - match t.as_ref().map(SerdeType::from) { + match t { Some(Struct(fields) | StructTuple(fields)) => { Self::Collection(ser, fields.iter().map(|f| f.ty().type_info()).collect()) } Some(Tuple(TupleOrArray::Array(ty, n))) => { - Self::Collection(ser, (0..n).map(|_| ty.type_info()).collect()) + Self::Collection(ser, (0..n).map(|_| ty.clone()).collect()) } - Some(Tuple(TupleOrArray::Tuple(fields))) => { - Self::Collection(ser, fields.iter().map(|f| f.type_info()).collect()) + Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Collection(ser, fields.into()), + Some(Variant(_, variants, _)) => { + // assuming the variant name is used to select the right variant + let variants = variants.iter().map(|v| *v.name()).collect(); + Self::Variant { + ser, + variants, + selected: None, + } } - Some(Variant(_, variants)) => Self::Variant(ser, variants.into()), _ => Self::Empty(ser), } } } -pub enum TypedSerializer<'a, W: 'a> { - Empty(&'a mut Serializer), - Collection(&'a mut Serializer, Vec), - Variant(&'a mut Serializer, Vec), +pub enum TypedSerializer<'a, B> { + Empty(&'a mut Serializer), + Collection(&'a mut Serializer, Vec), + Variant { + ser: &'a mut Serializer, + variants: Vec<&'a str>, + selected: Option, + }, } impl<'a, B> TypedSerializer<'a, B> { fn serializer(&mut self) -> &mut Serializer { match self { - Self::Empty(ser) | Self::Collection(ser, _) | Self::Variant(ser, _) => ser, + Self::Empty(ser) | Self::Collection(ser, _) | Self::Variant { ser, .. } => ser, } } } @@ -351,22 +359,34 @@ where where T: Serialize, { - if matches!(self, Self::Collection(_, _)) { - return Ok(()); - } match self { - TypedSerializer::Empty(_) => key.serialize(self.serializer()), - TypedSerializer::Collection(_, _) => Ok(()), - TypedSerializer::Variant(_, variants) => { + TypedSerializer::Collection(_, _) + | TypedSerializer::Variant { + selected: Some(_), .. + } => Ok(()), + TypedSerializer::Variant { + ser, + variants, + selected: selected @ None, + } => { let key_data = { let mut s = Serializer::new(vec![], None); key.serialize(&mut s)?; s.out }; - - println!("=========> {:?}: {:?}\n", key_data, variants); + let idx = variants + .iter() + .position(|name| { + let mut s = Serializer::new(vec![], None); + name.serialize(&mut s).expect("key serialized"); + s.out == key_data + }) + .expect("key exists") as u8; + idx.serialize(&mut **ser)?; + *selected = Some(idx); Ok(()) } + TypedSerializer::Empty(ser) => key.serialize(&mut **ser), } } @@ -374,14 +394,14 @@ where where T: Serialize, { - if let Self::Collection(ser, types) = self { - let mut ty = types.remove(0); + if let TypedSerializer::Collection(ser, types) = self { + let mut ty = types.remove(0).into(); // serde_json unwraps newtypes - if let SerdeType::StructNewType(t) = (&ty).into() { - ty = t + if let SerdeType::StructNewType(t) = ty { + ty = t.into() } ser.ty = Some(ty); - } + }; value.serialize(self.serializer()) } @@ -402,9 +422,9 @@ where T: Serialize, { if let Self::Collection(ser, types) = self { - let mut ty = types.remove(0); - if let SerdeType::StructNewType(t) = (&ty).into() { - ty = t + let mut ty = types.remove(0).into(); + if let SerdeType::StructNewType(t) = ty { + ty = t.into() } ser.ty = Some(ty); } @@ -572,7 +592,7 @@ mod tests { #[test] fn test_primitive_u8() -> Result<()> { let mut out = [0u8]; - to_writer(&mut out[..], &123u8)?; + to_bytes(&mut out[..], &123u8)?; let expected = [123]; @@ -586,7 +606,7 @@ mod tests { let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_ref(), expected); Ok(()) @@ -598,7 +618,7 @@ mod tests { let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_ref(), expected); Ok(()) @@ -610,7 +630,7 @@ mod tests { let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_mut(), expected); Ok(()) @@ -622,7 +642,7 @@ mod tests { let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_mut(), expected); Ok(()) @@ -634,7 +654,7 @@ mod tests { let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_mut(), expected); Ok(()) @@ -646,7 +666,7 @@ mod tests { let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_mut(), expected); Ok(()) @@ -658,7 +678,7 @@ mod tests { let mut out = [0u8]; let expected = INPUT.encode(); - to_writer(out.as_mut(), &INPUT)?; + to_bytes(out.as_mut(), &INPUT)?; assert_eq!(out.as_mut(), expected); Ok(()) @@ -670,7 +690,7 @@ mod tests { let mut out = Vec::::new(); let expected = INPUT.encode(); - to_writer(&mut out, &INPUT)?; + to_bytes(&mut out, &INPUT)?; assert_eq!(out, expected); Ok(()) @@ -682,7 +702,7 @@ mod tests { let mut out = Vec::::new(); let expected = INPUT.encode(); - to_writer(&mut out, &INPUT)?; + to_bytes(&mut out, &INPUT)?; assert_eq!(out, expected); Ok(()) @@ -694,7 +714,7 @@ mod tests { let mut out = Vec::::new(); let expected = INPUT.encode(); - to_writer(&mut out, &INPUT)?; + to_bytes(&mut out, &INPUT)?; assert_eq!(out, expected); Ok(()) @@ -712,7 +732,7 @@ mod tests { let mut out = Vec::::new(); let expected = INPUT.encode(); - to_writer(&mut out, &INPUT)?; + to_bytes(&mut out, &INPUT)?; assert_eq!(out, expected); Ok(()) @@ -730,7 +750,7 @@ mod tests { let mut out = Vec::::new(); let expected = input.encode(); - to_writer(&mut out, &input)?; + to_bytes(&mut out, &input)?; assert_eq!(out, expected); Ok(()) @@ -755,7 +775,7 @@ mod tests { let mut out = Vec::::new(); let expected = input.encode(); - to_writer(&mut out, &input)?; + to_bytes(&mut out, &input)?; assert_eq!(out, expected); Ok(()) @@ -767,7 +787,7 @@ mod tests { let mut out = Vec::::new(); let expected = input.encode(); - to_writer(&mut out, &input)?; + to_bytes(&mut out, &input)?; assert_eq!(out, expected); Ok(()) @@ -807,7 +827,7 @@ mod tests { let mut out = Vec::::new(); let expected = input.encode(); - to_writer(&mut out, &input)?; + to_bytes(&mut out, &input)?; assert_eq!(out, expected); Ok(()) @@ -835,7 +855,7 @@ mod tests { let json_input = to_value(&input).unwrap(); println!("{:?}\n", input); println!("{:?}\n", json_input); - to_writer_with_info(&mut out, &json_input, Foo::type_info())?; + to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; assert_eq!(out, expected); Ok(()) @@ -878,7 +898,7 @@ mod tests { let json_input = to_value(&input).unwrap(); println!("{:?}\n", input); println!("{:?}\n", json_input); - to_writer_with_info(&mut out, &json_input, Foo::type_info())?; + to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; assert_eq!(out, expected); Ok(()) diff --git a/src/value.rs b/src/value.rs index 82e634e..7dc401a 100644 --- a/src/value.rs +++ b/src/value.rs @@ -9,20 +9,22 @@ use serde::ser::{ }; use serde::Serialize; -/// A container for SCALE encoded data that can serialize types -/// directly with the help of a type registry and without using an -/// intermediate representation that requires allocating data. +/// A container for SCALE encoded data that can serialize types directly +/// with the help of a type registry and without using an intermediate representation. #[derive(Debug)] pub struct Value { data: Bytes, - ty: Type, + ty: SerdeType, + path: Vec<&'static str>, } impl Value { pub fn new(data: impl Into, ty: Type) -> Self { + let path = ty.path().segments().to_vec(); Value { data: data.into(), - ty, + ty: ty.into(), + path, } } @@ -33,7 +35,7 @@ impl Value { #[inline] fn ty_name(&self) -> &'static str { - self.ty.path().segments().last().copied().unwrap_or("") + self.path.last().copied().unwrap_or("") } } @@ -45,7 +47,7 @@ impl Serialize for Value { let mut data = self.data.clone(); use SerdeType::*; - match (&self.ty).into() { + match &self.ty { Bool => ser.serialize_bool(data.get_u8() != 0), U8 => ser.serialize_u8(data.get_u8()), U16 => ser.serialize_u16(data.get_u16_le()), @@ -89,7 +91,7 @@ impl Serialize for Value { Tuple(t) => { let mut state = ser.serialize_tuple(t.len())?; for i in 0..t.len() { - state.serialize_element(&Value::chunk(&mut data, t.type_info(i)))?; + state.serialize_element(&Value::chunk(&mut data, t.type_info(i).clone()))?; } state.end() } @@ -105,7 +107,7 @@ impl Serialize for Value { } StructUnit => ser.serialize_unit_struct(self.ty_name()), StructNewType(ty) => { - ser.serialize_newtype_struct(self.ty_name(), &Value::chunk(&mut data, ty)) + ser.serialize_newtype_struct(self.ty_name(), &Value::chunk(&mut data, ty.clone())) } StructTuple(fields) => { let mut state = ser.serialize_tuple_struct(self.ty_name(), fields.len())?; @@ -114,48 +116,47 @@ impl Serialize for Value { } state.end() } - ty @ Variant(_, _) => match ty.pick_variant(data.get_u8()) { - EnumVariant::OptionNone => ser.serialize_none(), - EnumVariant::OptionSome(ty) => ser.serialize_some(&Value::chunk(&mut data, ty)), - EnumVariant::Unit(v) => { - ser.serialize_unit_variant(self.ty_name(), v.index().into(), v.name()) - } - EnumVariant::NewType(v) => { - let ty = v.fields().first().unwrap().ty().type_info(); - ser.serialize_newtype_variant( + ty @ Variant(_, _, _) => { + let variant = &ty.pick(data.get_u8()); + match variant.into() { + EnumVariant::OptionNone => ser.serialize_none(), + EnumVariant::OptionSome(ty) => ser.serialize_some(&Value::chunk(&mut data, ty)), + EnumVariant::Unit(idx, ref name) => { + ser.serialize_unit_variant(self.ty_name(), idx.into(), name) + } + EnumVariant::NewType(idx, ref name, ty) => ser.serialize_newtype_variant( self.ty_name(), - v.index().into(), - v.name(), + idx.into(), + name, &Value::chunk(&mut data, ty), - ) - } - - EnumVariant::Tuple(v) => { - let mut s = ser.serialize_tuple_variant( - self.ty_name(), - v.index().into(), - v.name(), - v.fields().len(), - )?; - for f in v.fields() { - s.serialize_field(&Value::chunk(&mut data, f.ty().type_info()))?; + ), + + EnumVariant::Tuple(idx, ref name, fields) => { + let mut s = ser.serialize_tuple_variant( + self.ty_name(), + idx.into(), + name, + fields.len(), + )?; + for ty in fields { + s.serialize_field(&Value::chunk(&mut data, ty))?; + } + s.end() } - s.end() - } - EnumVariant::Struct(v) => { - let mut s = ser.serialize_struct_variant( - self.ty_name(), - v.index().into(), - v.name(), - v.fields().len(), - )?; - for f in v.fields() { - let ty = f.ty().type_info(); - s.serialize_field(f.name().unwrap(), &Value::chunk(&mut data, ty))?; + EnumVariant::Struct(idx, ref name, fields) => { + let mut s = ser.serialize_struct_variant( + self.ty_name(), + idx.into(), + name, + fields.len(), + )?; + for (fname, ty) in fields { + s.serialize_field(fname, &Value::chunk(&mut data, ty))?; + } + s.end() } - s.end() } - }, + } } } } From a9ee2209bc6ace894a0b37a82200b59e2bf0cd5b Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 1 Oct 2021 16:30:42 +0200 Subject: [PATCH 30/59] Cover most cases of serialization --- Cargo.toml | 8 +- src/lib.rs | 47 ++++++++++- src/serializer.rs | 199 ++++++++++++++++++++++------------------------ src/value.rs | 3 +- 4 files changed, 148 insertions(+), 109 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 0b32a16..ced6797 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,24 +1,24 @@ [package] name = "scales" description = "SCALE Serialization" -version = "0.3.0" +version = "0.5.0" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/valibre-org/scales" [dependencies] bytes = { version = "1.0.1", default-features = false } -scale-info = { version = "0.10.0", default-features = false } +scale-info = { version = "1.0.0", default-features = false } serde = { version = "1.0.126", default-features = false } [features] -default = ["std", "experimental-serializer"] +#default = ["std", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] [dev-dependencies] anyhow = "1.0.42" codec = { version = "2.2.0", package = "parity-scale-codec", features = ["derive"] } -scale-info = { version = "0.10.0", default-features = false, features = ["derive"] } +scale-info = { version = "1.0.0", default-features = false, features = ["derive"] } serde = { version = "1.0.126", default-features = false, features = ["derive"] } serde_json = { version = "1.0.66", default-features = false, features = ["alloc"] } diff --git a/src/lib.rs b/src/lib.rs index 5de8ea8..c558305 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,19 +1,30 @@ #![cfg_attr(not(feature = "std"), no_std)] +///! ///! # Scales ///! ///! Dynamic SCALE Serialization using `scale-info` type information. -///! +#[macro_use] +extern crate alloc; #[cfg(feature = "experimental-serializer")] mod serializer; mod value; #[cfg(feature = "experimental-serializer")] -pub use serializer::{to_bytes, Serializer}; +pub use serializer::{to_bytes, to_bytes_with_info, to_vec, to_vec_with_info, Serializer}; pub use value::Value; +use prelude::*; use scale_info::{Field, MetaType, Type, Variant}; +mod prelude { + pub use alloc::{ + collections::BTreeMap, + string::{String, ToString}, + vec::Vec, + }; +} + macro_rules! is_tuple { ($it:ident) => { $it.fields().first().and_then(Field::name).is_none() @@ -113,6 +124,7 @@ impl From for SerdeType { } } +// Utilities for enum variants impl SerdeType { fn pick(&self, index: u8) -> Self { match self { @@ -126,6 +138,37 @@ impl SerdeType { _ => panic!("Only for enum variants"), } } + + #[cfg(feature = "experimental-serializer")] + fn pick_mut(&mut self, selection: A, get_field: F) -> &Self + where + F: Fn(&Variant) -> B, + A: AsRef<[u8]> + PartialEq + core::fmt::Debug, + B: AsRef<[u8]> + PartialEq + core::fmt::Debug, + { + match self { + SerdeType::Variant(_, _, Some(_)) => self, + SerdeType::Variant(_, ref mut variants, idx @ None) => { + let i = variants + .iter() + .map(|v| get_field(v)) + .position(|f| f.as_ref() == selection.as_ref()) + .expect("index") as u8; + variants.retain(|v| v.index() == i); + *idx = Some(i); + self + } + _ => panic!("Only for enum variants"), + } + } + + #[cfg(feature = "experimental-serializer")] + fn variant_id(&self) -> u8 { + match self { + SerdeType::Variant(_, _, Some(id)) => *id, + _ => panic!("Only for enum variants"), + } + } } #[derive(Debug)] diff --git a/src/serializer.rs b/src/serializer.rs index 6d47b7f..c8abb82 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,33 +1,56 @@ +use crate::prelude::*; use bytes::BufMut; use core::fmt; use scale_info::{Type, TypeInfo}; use serde::{ser, Serialize}; -use crate::{SerdeType, TupleOrArray}; +use crate::{EnumVariant, SerdeType, TupleOrArray}; type Result = core::result::Result; #[derive(TypeInfo)] struct Noop; +#[inline] +pub fn to_vec(value: &T) -> Result> +where + T: Serialize + ?Sized, +{ + let mut out = vec![]; + to_bytes(&mut out, value)?; + Ok(out) +} + +#[inline] +pub fn to_vec_with_info(value: &T, info: impl Into>) -> Result> +where + T: Serialize + ?Sized, +{ + let mut out = vec![]; + to_bytes_with_info(&mut out, value, info)?; + Ok(out) +} + pub fn to_bytes(bytes: B, value: &T) -> Result<()> where - T: Serialize, + T: Serialize + ?Sized, B: BufMut, { to_bytes_with_info(bytes, value, None) } -pub fn to_bytes_with_info(writer: B, value: &T, info: impl Into>) -> Result<()> +pub fn to_bytes_with_info(bytes: B, value: &T, info: impl Into>) -> Result<()> where - T: Serialize, + T: Serialize + ?Sized, B: BufMut, { - let mut serializer = Serializer::new(writer, info); + let mut serializer = Serializer::new(bytes, info); value.serialize(&mut serializer)?; Ok(()) } +/// A serializer that encodes types to SCALE with the option to coerce +/// the output to an equivalent representation given by some type information. pub struct Serializer { out: B, ty: Option, @@ -168,13 +191,11 @@ where fn serialize_unit(self) -> Result { self.maybe_some()?; - println!("===== u {:?}\n", self.ty); Ok(()) } fn serialize_unit_struct(self, _name: &'static str) -> Result { self.maybe_some()?; - println!("===== us {:?}\n", self.ty); Ok(()) } @@ -185,7 +206,6 @@ where _variant: &'static str, ) -> Result { self.maybe_some()?; - println!("===== uv {:?}\n", self.ty); (variant_index as u8).serialize(self) } @@ -194,7 +214,6 @@ where T: Serialize, { self.maybe_some()?; - println!("===== ns {:?}\n", self.ty); value.serialize(self) } @@ -209,15 +228,13 @@ where T: Serialize, { self.maybe_some()?; - println!("===== nv {:?}\n", self.ty); self.out.put_u8(variant_index as u8); value.serialize(self) } fn serialize_seq(self, len: Option) -> Result { self.maybe_some()?; - println!("===== seq {:?}\n", self.ty); - if !matches!(self.ty, Some(SerdeType::StructTuple(_))) { + if matches!(self.ty, None | Some(SerdeType::Sequence(_))) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -225,7 +242,6 @@ where fn serialize_tuple(self, _len: usize) -> Result { self.maybe_some()?; - println!("===== tup {:?}\n", self.ty); Ok(self.into()) } @@ -235,7 +251,6 @@ where __len: usize, ) -> Result { self.maybe_some()?; - println!("===== tups {:?}", self.ty); Ok(self.into()) } @@ -248,18 +263,12 @@ where ) -> Result { self.maybe_some()?; self.out.put_u8(variant_index as u8); - println!("===== tupv {:?}", self.ty); Ok(self.into()) } fn serialize_map(self, len: Option) -> Result { self.maybe_some()?; - println!("===== m {:?}\n", self.ty); - if matches!(self.ty, Some(SerdeType::Variant(_, _, _))) { - // TODO! - println!("map as variant: {:?}\n", len); - } - if !matches!(self.ty, Some(SerdeType::Struct(_))) { + if matches!(self.ty, None | Some(SerdeType::Map(_, _))) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -267,7 +276,6 @@ where fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { self.maybe_some()?; - println!("===== s {:?}", self.ty); Ok(self.into()) } @@ -280,7 +288,6 @@ where ) -> Result { self.maybe_some()?; self.out.put_u8(variant_index as u8); - println!("===== sv {:?}", self.ty); Ok(self.into()) } } @@ -303,47 +310,51 @@ where } } +/// +pub enum TypedSerializer<'a, B> { + Empty(&'a mut Serializer), + Composite(&'a mut Serializer, Vec), + Sequence(&'a mut Serializer, Type), + Enum(&'a mut Serializer), +} + impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { fn from(ser: &'a mut Serializer) -> Self { use SerdeType::*; - - let t = ser.ty.take(); - match t { + match ser.ty.take() { Some(Struct(fields) | StructTuple(fields)) => { - Self::Collection(ser, fields.iter().map(|f| f.ty().type_info()).collect()) + Self::Composite(ser, fields.iter().map(|f| f.ty().type_info()).collect()) } - Some(Tuple(TupleOrArray::Array(ty, n))) => { - Self::Collection(ser, (0..n).map(|_| ty.clone()).collect()) + Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty.clone()), + Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Composite(ser, fields.into()), + Some(Sequence(ty)) => { + ser.ty = Some(ty.clone().into()); + Self::Sequence(ser, ty) } - Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Collection(ser, fields.into()), - Some(Variant(_, variants, _)) => { - // assuming the variant name is used to select the right variant - let variants = variants.iter().map(|v| *v.name()).collect(); - Self::Variant { - ser, - variants, - selected: None, + Some(Map(_, _)) => Self::Empty(ser), + Some(var @ Variant(_, _, Some(_))) => match (&var).into() { + EnumVariant::Tuple(_, _, types) => Self::Composite(ser, types), + EnumVariant::Struct(_, _, types) => { + Self::Composite(ser, types.iter().map(|(_, ty)| ty.clone()).collect()) } + _ => Self::Empty(ser), + }, + Some(var @ Variant(_, _, None)) => { + ser.ty = Some(var); + Self::Enum(ser) } _ => Self::Empty(ser), } } } -pub enum TypedSerializer<'a, B> { - Empty(&'a mut Serializer), - Collection(&'a mut Serializer, Vec), - Variant { - ser: &'a mut Serializer, - variants: Vec<&'a str>, - selected: Option, - }, -} - impl<'a, B> TypedSerializer<'a, B> { fn serializer(&mut self) -> &mut Serializer { match self { - Self::Empty(ser) | Self::Collection(ser, _) | Self::Variant { ser, .. } => ser, + Self::Empty(ser) + | Self::Composite(ser, _) + | Self::Enum(ser) + | Self::Sequence(ser, _) => ser, } } } @@ -360,33 +371,18 @@ where T: Serialize, { match self { - TypedSerializer::Collection(_, _) - | TypedSerializer::Variant { - selected: Some(_), .. - } => Ok(()), - TypedSerializer::Variant { - ser, - variants, - selected: selected @ None, - } => { - let key_data = { - let mut s = Serializer::new(vec![], None); - key.serialize(&mut s)?; - s.out - }; - let idx = variants - .iter() - .position(|name| { - let mut s = Serializer::new(vec![], None); - name.serialize(&mut s).expect("key serialized"); - s.out == key_data - }) - .expect("key exists") as u8; - idx.serialize(&mut **ser)?; - *selected = Some(idx); + TypedSerializer::Enum(ser) => { + if let Some(ref mut var @ SerdeType::Variant(_, _, None)) = ser.ty { + let key_data = to_vec(key)?; + // assume the key is the name of the variant + var.pick_mut(key_data, |v| to_vec(v.name()).unwrap()) + .variant_id() + .serialize(&mut **ser)?; + } Ok(()) } TypedSerializer::Empty(ser) => key.serialize(&mut **ser), + _ => Ok(()), } } @@ -394,7 +390,7 @@ where where T: Serialize, { - if let TypedSerializer::Collection(ser, types) = self { + if let TypedSerializer::Composite(ser, types) = self { let mut ty = types.remove(0).into(); // serde_json unwraps newtypes if let SerdeType::StructNewType(t) = ty { @@ -421,12 +417,15 @@ where where T: Serialize, { - if let Self::Collection(ser, types) = self { - let mut ty = types.remove(0).into(); - if let SerdeType::StructNewType(t) = ty { - ty = t.into() + match self { + TypedSerializer::Composite(ser, types) => { + let mut ty = types.remove(0).into(); + if let SerdeType::StructNewType(t) = ty { + ty = t.into() + } + ser.ty = Some(ty); } - ser.ty = Some(ty); + _ => {} } value.serialize(self.serializer()) } @@ -587,10 +586,9 @@ mod tests { use core::mem::size_of; use scale_info::TypeInfo; use serde_json::to_value; - use std::collections::BTreeMap; #[test] - fn test_primitive_u8() -> Result<()> { + fn primitive_u8() -> Result<()> { let mut out = [0u8]; to_bytes(&mut out[..], &123u8)?; @@ -601,7 +599,7 @@ mod tests { } #[test] - fn test_primitive_u16() -> Result<()> { + fn primitive_u16() -> Result<()> { const INPUT: u16 = 0xFF_EE; let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); @@ -613,7 +611,7 @@ mod tests { } #[test] - fn test_primitive_u32() -> Result<()> { + fn primitive_u32() -> Result<()> { const INPUT: u32 = 0xFF_EE_DD_CC; let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); @@ -625,7 +623,7 @@ mod tests { } #[test] - fn test_primitive_u64() -> Result<()> { + fn primitive_u64() -> Result<()> { const INPUT: u64 = 0xFF_EE_DD_CC__BB_AA_99_88; let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); @@ -637,7 +635,7 @@ mod tests { } #[test] - fn test_primitive_i16() -> Result<()> { + fn primitive_i16() -> Result<()> { const INPUT: i16 = i16::MIN; let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); @@ -649,7 +647,7 @@ mod tests { } #[test] - fn test_primitive_i32() -> Result<()> { + fn primitive_i32() -> Result<()> { const INPUT: i32 = i32::MIN; let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); @@ -661,7 +659,7 @@ mod tests { } #[test] - fn test_primitive_i64() -> Result<()> { + fn primitive_i64() -> Result<()> { const INPUT: i64 = i64::MIN; let mut out = [0u8; size_of::()]; let expected = INPUT.encode(); @@ -673,7 +671,7 @@ mod tests { } #[test] - fn test_primitive_bool() -> Result<()> { + fn primitive_bool() -> Result<()> { const INPUT: bool = true; let mut out = [0u8]; let expected = INPUT.encode(); @@ -685,7 +683,7 @@ mod tests { } #[test] - fn test_str() -> Result<()> { + fn str() -> Result<()> { const INPUT: &str = "ac orci phasellus egestas tellus rutrum tellus pellentesque"; let mut out = Vec::::new(); let expected = INPUT.encode(); @@ -697,7 +695,7 @@ mod tests { } #[test] - fn test_bytes() -> Result<()> { + fn bytes() -> Result<()> { const INPUT: &[u8] = b"dictumst quisque sagittis purus sit amet volutpat consequat"; let mut out = Vec::::new(); let expected = INPUT.encode(); @@ -709,7 +707,7 @@ mod tests { } #[test] - fn test_tuple_simple() -> Result<()> { + fn tuple_simple() -> Result<()> { const INPUT: (u8, bool, u64) = (0xD0, false, u64::MAX); let mut out = Vec::::new(); let expected = INPUT.encode(); @@ -721,7 +719,7 @@ mod tests { } #[test] - fn test_enum_simple() -> Result<()> { + fn enum_simple() -> Result<()> { #[derive(Serialize, Encode)] enum X { _A, @@ -739,7 +737,7 @@ mod tests { } #[test] - fn test_tuple_enum_mix() -> Result<()> { + fn tuple_enum_mix() -> Result<()> { #[derive(Serialize, Encode)] enum X { A, @@ -757,7 +755,7 @@ mod tests { } #[test] - fn test_struct_simple() -> Result<()> { + fn struct_simple() -> Result<()> { #[derive(Serialize, Encode)] struct Foo { a: Bar, @@ -782,7 +780,7 @@ mod tests { } #[test] - fn test_vec_simple() -> Result<()> { + fn vec_simple() -> Result<()> { let input: Vec = vec!["hello".into(), "beautiful".into(), "people".into()]; let mut out = Vec::::new(); let expected = input.encode(); @@ -794,7 +792,7 @@ mod tests { } #[test] - fn test_struct_mix() -> Result<()> { + fn struct_mix() -> Result<()> { #[derive(Serialize, Encode)] struct Foo<'a> { a: Vec, @@ -834,7 +832,7 @@ mod tests { } #[test] - fn test_json_simple() -> Result<()> { + fn json_simple() -> Result<()> { #[derive(Debug, Serialize, Encode, TypeInfo)] struct Foo { a: Bar, @@ -853,8 +851,6 @@ mod tests { let expected = input.encode(); let json_input = to_value(&input).unwrap(); - println!("{:?}\n", input); - println!("{:?}\n", json_input); to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; assert_eq!(out, expected); @@ -862,7 +858,7 @@ mod tests { } #[test] - fn test_json_mix() -> Result<()> { + fn json_mix() -> Result<()> { #[derive(Debug, Serialize, Encode, TypeInfo)] struct Foo<'a> { a: Vec, @@ -885,7 +881,8 @@ mod tests { Bar::C( { let mut h = BTreeMap::new(); - h.insert("key".into(), false); + h.insert("key1".into(), false); + h.insert("key2".into(), true); h }, i64::MIN, @@ -896,8 +893,6 @@ mod tests { let expected = input.encode(); let json_input = to_value(&input).unwrap(); - println!("{:?}\n", input); - println!("{:?}\n", json_input); to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; assert_eq!(out, expected); diff --git a/src/value.rs b/src/value.rs index 7dc401a..fcac5b1 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,4 +1,5 @@ use crate::{EnumVariant, SerdeType}; +use alloc::vec::Vec; use bytes::{Buf, Bytes}; use core::convert::TryInto; use core::str; @@ -244,7 +245,7 @@ fn sequence_len(data: &[u8]) -> (usize, usize) { #[cfg(test)] mod tests { - use std::collections::BTreeMap; + use alloc::collections::BTreeMap; use super::*; use anyhow::Error; From ad64b1643f5a846098c5651d0e9440b85b657f31 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 4 Oct 2021 14:50:32 +0200 Subject: [PATCH 31/59] Serializer support more cases with unit and newtypes --- Cargo.toml | 6 +-- src/lib.rs | 20 +++++++-- src/serializer.rs | 104 ++++++++++++++++++++++++++++++++++++---------- 3 files changed, 100 insertions(+), 30 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ced6797..89ff682 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,10 @@ [package] name = "scales" description = "SCALE Serialization" -version = "0.5.0" +version = "0.5.1" authors = ["Daniel Olano "] edition = "2018" -repository = "https://github.com/valibre-org/scales" +repository = "https://github.com/virto-network/scales" [dependencies] bytes = { version = "1.0.1", default-features = false } @@ -12,7 +12,7 @@ scale-info = { version = "1.0.0", default-features = false } serde = { version = "1.0.126", default-features = false } [features] -#default = ["std", "experimental-serializer"] +default = ["std", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] diff --git a/src/lib.rs b/src/lib.rs index c558305..eb19732 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,8 +34,8 @@ macro_rules! is_tuple { /// A convenient representation of the scale-info types to a format /// that matches serde model more closely #[rustfmt::skip] -#[derive(Debug)] -enum SerdeType { +#[derive(Debug, Clone)] +pub enum SerdeType { Bool, U8, U16, U32, U64, U128, I8, I16, I32, I64, I128, @@ -49,6 +49,18 @@ enum SerdeType { Variant(String, Vec, Option), } +impl From<&mut Type> for SerdeType { + fn from(ty: &mut Type) -> Self { + ty.clone().into() + } +} + +impl From<&Type> for SerdeType { + fn from(ty: &Type) -> Self { + ty.clone().into() + } +} + impl From for SerdeType { fn from(ty: Type) -> Self { use scale_info::{TypeDef, TypeDef::*, TypeDefComposite, TypeDefPrimitive}; @@ -220,8 +232,8 @@ impl<'a> From<&SerdeType> for EnumVariant<'a> { } } -#[derive(Debug)] -enum TupleOrArray { +#[derive(Debug, Clone)] +pub enum TupleOrArray { Array(Type, u32), Tuple(Vec), } diff --git a/src/serializer.rs b/src/serializer.rs index c8abb82..7c6944a 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -139,7 +139,11 @@ where } fn serialize_u64(self, v: u64) -> Result { + // all numbers in serde_json are the same match self.ty { + Some(SerdeType::I8) => self.serialize_i8(v as i8)?, + Some(SerdeType::I16) => self.serialize_i16(v as i16)?, + Some(SerdeType::I32) => self.serialize_i32(v as i32)?, Some(SerdeType::U8) => self.serialize_u8(v as u8)?, Some(SerdeType::U16) => self.serialize_u16(v as u16)?, Some(SerdeType::U32) => self.serialize_u32(v as u32)?, @@ -165,6 +169,11 @@ where fn serialize_str(self, v: &str) -> Result { self.maybe_some()?; + if let Some(ref mut var @ SerdeType::Variant(_, _, None)) = &mut self.ty { + var.pick_mut(to_vec(v)?, |k| to_vec(k.name()).unwrap()); + self.out.put_u8(var.variant_id()); + return Ok(()); + } compact_number(v.len(), &mut self.out); self.out.put(v.as_bytes()); Ok(()) @@ -313,8 +322,8 @@ where /// pub enum TypedSerializer<'a, B> { Empty(&'a mut Serializer), - Composite(&'a mut Serializer, Vec), - Sequence(&'a mut Serializer, Type), + Composite(&'a mut Serializer, Vec), + Sequence(&'a mut Serializer, SerdeType), Enum(&'a mut Serializer), } @@ -322,20 +331,22 @@ impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { fn from(ser: &'a mut Serializer) -> Self { use SerdeType::*; match ser.ty.take() { - Some(Struct(fields) | StructTuple(fields)) => { - Self::Composite(ser, fields.iter().map(|f| f.ty().type_info()).collect()) - } - Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty.clone()), - Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Composite(ser, fields.into()), - Some(Sequence(ty)) => { - ser.ty = Some(ty.clone().into()); - Self::Sequence(ser, ty) + Some(Struct(fields) | StructTuple(fields)) => Self::Composite( + ser, + fields.iter().map(|f| f.ty().type_info().into()).collect(), + ), + Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty.into()), + Some(Tuple(TupleOrArray::Tuple(fields))) => { + Self::Composite(ser, fields.iter().map(SerdeType::from).collect()) } + Some(Sequence(ty)) => Self::Sequence(ser, ty.into()), Some(Map(_, _)) => Self::Empty(ser), Some(var @ Variant(_, _, Some(_))) => match (&var).into() { - EnumVariant::Tuple(_, _, types) => Self::Composite(ser, types), + EnumVariant::Tuple(_, _, types) => { + Self::Composite(ser, types.iter().map(SerdeType::from).collect()) + } EnumVariant::Struct(_, _, types) => { - Self::Composite(ser, types.iter().map(|(_, ty)| ty.clone()).collect()) + Self::Composite(ser, types.iter().map(|(_, ty)| ty.into()).collect()) } _ => Self::Empty(ser), }, @@ -390,14 +401,24 @@ where where T: Serialize, { - if let TypedSerializer::Composite(ser, types) = self { - let mut ty = types.remove(0).into(); - // serde_json unwraps newtypes - if let SerdeType::StructNewType(t) = ty { - ty = t.into() + match self { + TypedSerializer::Composite(ser, types) => { + let mut ty = types.remove(0); + // serde_json unwraps newtypes + if let SerdeType::StructNewType(t) = ty { + ty = t.into() + } + ser.ty = Some(ty); } - ser.ty = Some(ty); - }; + TypedSerializer::Enum(ser) => { + if let Some(var @ SerdeType::Variant(_, _, Some(_))) = &ser.ty { + if let EnumVariant::NewType(_, _, ty) = var.into() { + ser.ty = Some(ty.into()); + } + } + } + _ => {} + } value.serialize(self.serializer()) } @@ -420,13 +441,20 @@ where match self { TypedSerializer::Composite(ser, types) => { let mut ty = types.remove(0).into(); - if let SerdeType::StructNewType(t) = ty { - ty = t.into() - } + match ty { + SerdeType::StructNewType(t) => ty = t.into(), + _ => {} + }; ser.ty = Some(ty); } + TypedSerializer::Sequence(ser, ty) => { + ser.ty = Some(match ty { + SerdeType::StructNewType(t) => t.into(), + _ => ty.clone(), + }); + } _ => {} - } + }; value.serialize(self.serializer()) } @@ -898,4 +926,34 @@ mod tests { assert_eq!(out, expected); Ok(()) } + + #[test] + fn json_mix2() -> Result<()> { + #[derive(Debug, Encode, Serialize, TypeInfo)] + enum Bar { + This, + That(i16), + } + #[derive(Debug, Encode, Serialize, TypeInfo)] + struct Baz(String); + #[derive(Debug, Encode, Serialize, TypeInfo)] + struct Foo { + bar: Vec, + baz: Option, + lol: &'static [u8], + } + let input = Foo { + bar: [Bar::That(i16::MAX), Bar::This].into(), + baz: Some(Baz("lorem ipsum".into())), + lol: b"\0xFFsome stuff\0x00", + }; + let mut out = Vec::::new(); + let expected = input.encode(); + + let json_input = to_value(&input).unwrap(); + to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; + + assert_eq!(out, expected); + Ok(()) + } } From f15ece842ee49d805eed2a49cb8b9f2aa530f02b Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 4 Oct 2021 15:12:33 +0200 Subject: [PATCH 32/59] Fix clippy warnings --- Cargo.toml | 5 +++-- src/lib.rs | 2 +- src/serializer.rs | 38 +++++++++++++++++--------------------- src/value.rs | 8 ++++---- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 89ff682..6b9ffb5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,11 @@ [package] -name = "scales" +name = "scale-serialization" description = "SCALE Serialization" -version = "0.5.1" +version = "0.5.2" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" +license = "Apache-2.0" [dependencies] bytes = { version = "1.0.1", default-features = false } diff --git a/src/lib.rs b/src/lib.rs index eb19732..8318aa5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -103,7 +103,7 @@ impl From for SerdeType { Variant(v) => Self::Variant(name.into(), v.variants().into(), None), Sequence(s) => { let ty = s.type_param().type_info(); - if ty.path().segments() != &["u8"] { + if ty.path().segments() != ["u8"] { Self::Sequence(ty) } else { Self::Bytes diff --git a/src/serializer.rs b/src/serializer.rs index 7c6944a..fc70bce 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -322,8 +322,8 @@ where /// pub enum TypedSerializer<'a, B> { Empty(&'a mut Serializer), - Composite(&'a mut Serializer, Vec), - Sequence(&'a mut Serializer, SerdeType), + Composite(&'a mut Serializer, Vec), + Sequence(&'a mut Serializer, Type), Enum(&'a mut Serializer), } @@ -331,22 +331,17 @@ impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { fn from(ser: &'a mut Serializer) -> Self { use SerdeType::*; match ser.ty.take() { - Some(Struct(fields) | StructTuple(fields)) => Self::Composite( - ser, - fields.iter().map(|f| f.ty().type_info().into()).collect(), - ), - Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty.into()), - Some(Tuple(TupleOrArray::Tuple(fields))) => { - Self::Composite(ser, fields.iter().map(SerdeType::from).collect()) + Some(Struct(fields) | StructTuple(fields)) => { + Self::Composite(ser, fields.iter().map(|f| f.ty().type_info()).collect()) } - Some(Sequence(ty)) => Self::Sequence(ser, ty.into()), + Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty), + Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Composite(ser, fields), + Some(Sequence(ty)) => Self::Sequence(ser, ty), Some(Map(_, _)) => Self::Empty(ser), Some(var @ Variant(_, _, Some(_))) => match (&var).into() { - EnumVariant::Tuple(_, _, types) => { - Self::Composite(ser, types.iter().map(SerdeType::from).collect()) - } + EnumVariant::Tuple(_, _, types) => Self::Composite(ser, types), EnumVariant::Struct(_, _, types) => { - Self::Composite(ser, types.iter().map(|(_, ty)| ty.into()).collect()) + Self::Composite(ser, types.iter().map(|(_, ty)| ty.clone()).collect()) } _ => Self::Empty(ser), }, @@ -403,7 +398,7 @@ where { match self { TypedSerializer::Composite(ser, types) => { - let mut ty = types.remove(0); + let mut ty = types.remove(0).into(); // serde_json unwraps newtypes if let SerdeType::StructNewType(t) = ty { ty = t.into() @@ -441,16 +436,15 @@ where match self { TypedSerializer::Composite(ser, types) => { let mut ty = types.remove(0).into(); - match ty { - SerdeType::StructNewType(t) => ty = t.into(), - _ => {} - }; + if let SerdeType::StructNewType(t) = ty { + ty = t.into(); + } ser.ty = Some(ty); } TypedSerializer::Sequence(ser, ty) => { - ser.ty = Some(match ty { + ser.ty = Some(match ty.into() { SerdeType::StructNewType(t) => t.into(), - _ => ty.clone(), + _ => ty.into(), }); } _ => {} @@ -582,6 +576,8 @@ impl ser::Error for Error { } } +// from https://github.com/paritytech/parity-scale-codec/blob/master/src/compact.rs#L336 +#[allow(clippy::all)] fn compact_number(n: usize, mut dest: impl BufMut) { match n { 0..=0b0011_1111 => dest.put_u8((n as u8) << 2), diff --git a/src/value.rs b/src/value.rs index fcac5b1..7a367be 100644 --- a/src/value.rs +++ b/src/value.rs @@ -122,17 +122,17 @@ impl Serialize for Value { match variant.into() { EnumVariant::OptionNone => ser.serialize_none(), EnumVariant::OptionSome(ty) => ser.serialize_some(&Value::chunk(&mut data, ty)), - EnumVariant::Unit(idx, ref name) => { + EnumVariant::Unit(idx, name) => { ser.serialize_unit_variant(self.ty_name(), idx.into(), name) } - EnumVariant::NewType(idx, ref name, ty) => ser.serialize_newtype_variant( + EnumVariant::NewType(idx, name, ty) => ser.serialize_newtype_variant( self.ty_name(), idx.into(), name, &Value::chunk(&mut data, ty), ), - EnumVariant::Tuple(idx, ref name, fields) => { + EnumVariant::Tuple(idx, name, fields) => { let mut s = ser.serialize_tuple_variant( self.ty_name(), idx.into(), @@ -144,7 +144,7 @@ impl Serialize for Value { } s.end() } - EnumVariant::Struct(idx, ref name, fields) => { + EnumVariant::Struct(idx, name, fields) => { let mut s = ser.serialize_struct_variant( self.ty_name(), idx.into(), From 787901766417c05998a4567b29053039096899ba Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 4 Oct 2021 15:54:50 +0200 Subject: [PATCH 33/59] Update bytes --- Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6b9ffb5..8037061 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,14 +1,14 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.5.2" +version = "0.5.3" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" license = "Apache-2.0" [dependencies] -bytes = { version = "1.0.1", default-features = false } +bytes = { version = "1.1.0", default-features = false } scale-info = { version = "1.0.0", default-features = false } serde = { version = "1.0.126", default-features = false } From d4ef17ea2e5e4f1447d585c5ac9a2d7dc413311b Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 15 Oct 2021 13:32:04 +0200 Subject: [PATCH 34/59] Change to serialize value from portable format --- Cargo.toml | 2 +- src/lib.rs | 141 +++++++++------- src/value.rs | 458 +++++++++++++++++++++++++++------------------------ 3 files changed, 328 insertions(+), 273 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8037061..3730c7e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ scale-info = { version = "1.0.0", default-features = false } serde = { version = "1.0.126", default-features = false } [features] -default = ["std", "experimental-serializer"] +default = ["std"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] diff --git a/src/lib.rs b/src/lib.rs index 8318aa5..330d047 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,7 +15,7 @@ pub use serializer::{to_bytes, to_bytes_with_info, to_vec, to_vec_with_info, Ser pub use value::Value; use prelude::*; -use scale_info::{Field, MetaType, Type, Variant}; +use scale_info::{form::PortableForm as Portable, PortableRegistry}; mod prelude { pub use alloc::{ @@ -25,6 +25,11 @@ mod prelude { }; } +type Type = scale_info::Type; +type Field = scale_info::Field; +type Variant = scale_info::Variant; +type TypeId = u32; + macro_rules! is_tuple { ($it:ident) => { $it.fields().first().and_then(Field::name).is_none() @@ -42,78 +47,92 @@ pub enum SerdeType { Bytes, Char, Str, - Sequence(Type), - Map(Type, Type), + Sequence(TypeId), + Map(TypeId, TypeId), Tuple(TupleOrArray), - Struct(Vec), StructUnit, StructNewType(Type), StructTuple(Vec), + Struct(Vec<(String, TypeId)>), StructUnit, StructNewType(TypeId), StructTuple(Vec), Variant(String, Vec, Option), } -impl From<&mut Type> for SerdeType { - fn from(ty: &mut Type) -> Self { - ty.clone().into() - } -} +// impl From<&mut Type> for SerdeType { +// fn from(ty: &mut Type) -> Self { +// ty.clone().into() +// } +// } -impl From<&Type> for SerdeType { - fn from(ty: &Type) -> Self { - ty.clone().into() - } -} +// impl From<&Type> for SerdeType { +// fn from(ty: &Type) -> Self { +// ty.clone().into() +// } +// } -impl From for SerdeType { - fn from(ty: Type) -> Self { - use scale_info::{TypeDef, TypeDef::*, TypeDefComposite, TypeDefPrimitive}; - let name = ty.path().segments().last().copied().unwrap_or(""); +impl From<(&Type, &PortableRegistry)> for SerdeType { + fn from((ty, registry): (&Type, &PortableRegistry)) -> Self { + use scale_info::{TypeDef, TypeDefComposite, TypeDefPrimitive}; + type Def = TypeDef; - #[inline] - fn is_map(ty: &Type) -> bool { - ty.path().segments() == ["BTreeMap"] + macro_rules! resolve { + ($ty:expr) => { + registry.resolve($ty.id()).unwrap() + }; } - fn map_types(ty: &TypeDefComposite) -> (Type, Type) { + let is_map = |ty: &Type| -> bool { ty.path().segments() == ["BTreeMap"] }; + let map_types = |ty: &TypeDefComposite| -> (TypeId, TypeId) { let field = ty.fields().first().expect("map"); // Type information of BTreeMap is weirdly packed - if let TypeDef::Sequence(s) = field.ty().type_info().type_def() { - if let TypeDef::Tuple(t) = s.type_param().type_info().type_def() { + if let Def::Sequence(s) = resolve!(field.ty()).type_def() { + if let Def::Tuple(t) = resolve!(s.type_param()).type_def() { assert_eq!(t.fields().len(), 2); - let key_ty = t.fields().first().expect("key").type_info(); - let val_ty = t.fields().last().expect("val").type_info(); + let key_ty = t.fields().first().expect("key").id(); + let val_ty = t.fields().last().expect("val").id(); return (key_ty, val_ty); } } unreachable!() - } + }; + + let name = ty + .path() + .segments() + .last() + .cloned() + .unwrap_or_else(String::new); match ty.type_def() { - Composite(c) => { + Def::Composite(c) => { let fields = c.fields(); if fields.is_empty() { Self::StructUnit - } else if is_map(&ty) { + } else if is_map(ty) { let (k, v) = map_types(c); Self::Map(k, v) } else if fields.len() == 1 { - Self::StructNewType(fields.first().unwrap().ty().type_info()) + Self::StructNewType(fields.first().unwrap().ty().id()) } else if is_tuple!(c) { - Self::StructTuple(fields.into()) + Self::StructTuple(fields.iter().map(|f| f.ty().id()).collect()) } else { - Self::Struct(fields.into()) + Self::Struct( + fields + .iter() + .map(|f| (f.name().unwrap().into(), f.ty().id())) + .collect(), + ) } } - Variant(v) => Self::Variant(name.into(), v.variants().into(), None), - Sequence(s) => { - let ty = s.type_param().type_info(); + Def::Variant(v) => Self::Variant(name.into(), v.variants().into(), None), + Def::Sequence(s) => { + let ty = resolve!(s.type_param()); if ty.path().segments() != ["u8"] { - Self::Sequence(ty) + Self::Sequence(s.type_param().id()) } else { Self::Bytes } } - Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param().type_info(), a.len())), - Tuple(t) => Self::Tuple(TupleOrArray::Tuple( - t.fields().iter().map(MetaType::type_info).collect(), + Def::Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param().id(), a.len())), + Def::Tuple(t) => Self::Tuple(TupleOrArray::Tuple( + t.fields().iter().map(|ty| ty.id()).collect(), )), - Primitive(p) => match p { + Def::Primitive(p) => match p { TypeDefPrimitive::U8 => Self::U8, TypeDefPrimitive::U16 => Self::U16, TypeDefPrimitive::U32 => Self::U32, @@ -130,8 +149,8 @@ impl From for SerdeType { TypeDefPrimitive::U256 => unimplemented!(), TypeDefPrimitive::I256 => unimplemented!(), }, - Compact(_c) => todo!(), - BitSequence(_b) => todo!(), + Def::Compact(_c) => todo!(), + Def::BitSequence(_b) => todo!(), } } } @@ -186,45 +205,45 @@ impl SerdeType { #[derive(Debug)] enum EnumVariant<'a> { OptionNone, - OptionSome(Type), + OptionSome(TypeId), Unit(u8, &'a str), - NewType(u8, &'a str, Type), - Tuple(u8, &'a str, Vec), - Struct(u8, &'a str, Vec<(&'a str, Type)>), + NewType(u8, &'a str, TypeId), + Tuple(u8, &'a str, Vec), + Struct(u8, &'a str, Vec<(&'a str, TypeId)>), } -impl<'a> From<&SerdeType> for EnumVariant<'a> { - fn from(ty: &SerdeType) -> Self { +impl<'a> From<&'a SerdeType> for EnumVariant<'a> { + fn from(ty: &'a SerdeType) -> Self { match ty { SerdeType::Variant(name, variants, Some(idx)) => { let variant = variants.first().expect("single variant"); let fields = variant.fields(); - let vname = *variant.name(); + let vname = &*variant.name(); if fields.is_empty() { if name == "Option" && vname == "None" { Self::OptionNone } else { - Self::Unit(*idx, vname) + Self::Unit(*idx, vname.as_str()) } } else if is_tuple!(variant) { if fields.len() == 1 { - let ty = fields.first().map(|f| f.ty().type_info()).unwrap(); + let ty = fields.first().map(|f| f.ty().id()).unwrap(); return if name == "Option" && variant.name() == &"Some" { Self::OptionSome(ty) } else { - Self::NewType(*idx, vname, ty) + Self::NewType(*idx, vname.as_str(), ty) }; } else { - let fields = fields.iter().map(|f| f.ty().type_info()).collect(); - Self::Tuple(*idx, vname, fields) + let fields = fields.iter().map(|f| f.ty().id()).collect(); + Self::Tuple(*idx, vname.as_str(), fields) } } else { let fields = fields .iter() - .map(|f| (*f.name().unwrap(), f.ty().type_info())) + .map(|f| (f.name().unwrap().as_str(), f.ty().id())) .collect(); - Self::Struct(*idx, vname, fields) + Self::Struct(*idx, vname.as_str(), fields) } } _ => panic!("Only for enum variants"), @@ -234,8 +253,8 @@ impl<'a> From<&SerdeType> for EnumVariant<'a> { #[derive(Debug, Clone)] pub enum TupleOrArray { - Array(Type, u32), - Tuple(Vec), + Array(TypeId, u32), + Tuple(Vec), } impl TupleOrArray { fn len(&self) -> usize { @@ -245,10 +264,10 @@ impl TupleOrArray { } } - fn type_info(&self, i: usize) -> &Type { + fn type_id(&self, i: usize) -> TypeId { match self { - Self::Array(ty, _) => ty, - Self::Tuple(fields) => &fields[i], + Self::Array(ty, _) => *ty, + Self::Tuple(fields) => fields[i], } } } diff --git a/src/value.rs b/src/value.rs index 7a367be..cb37a0f 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,54 +1,107 @@ use crate::{EnumVariant, SerdeType}; -use alloc::vec::Vec; +use alloc::collections::BTreeMap; use bytes::{Buf, Bytes}; use core::convert::TryInto; use core::str; -use scale_info::{prelude::*, Type, TypeDef, TypeDefPrimitive as Primitive}; -use serde::ser::{ - SerializeMap, SerializeSeq, SerializeStruct, SerializeStructVariant, SerializeTuple, - SerializeTupleStruct, SerializeTupleVariant, -}; +use scale_info::{prelude::*, PortableRegistry, TypeDefPrimitive as Primitive}; +use serde::ser::{SerializeMap, SerializeSeq, SerializeTuple, SerializeTupleStruct}; use serde::Serialize; +type Type = scale_info::Type; +type TypeId = u32; +type TypeDef = scale_info::TypeDef; + /// A container for SCALE encoded data that can serialize types directly /// with the help of a type registry and without using an intermediate representation. #[derive(Debug)] -pub struct Value { +pub struct Value<'a> { data: Bytes, - ty: SerdeType, - path: Vec<&'static str>, + ty_id: TypeId, + registry: &'a PortableRegistry, } -impl Value { - pub fn new(data: impl Into, ty: Type) -> Self { - let path = ty.path().segments().to_vec(); +impl<'a> Value<'a> { + pub fn new(data: impl Into, ty_id: u32, registry: &'a PortableRegistry) -> Self { Value { data: data.into(), - ty: ty.into(), - path, + ty_id, + registry, } } - fn chunk(mut data: impl Buf, ty: Type) -> Self { - let size = type_len(&ty, data.chunk()); - Value::new(data.copy_to_bytes(size), ty) + fn new_value(&self, data: &mut Bytes, ty_id: TypeId) -> Self { + let size = self.ty_len(data.chunk(), ty_id); + Value::new(data.copy_to_bytes(size), ty_id, self.registry) } #[inline] - fn ty_name(&self) -> &'static str { - self.path.last().copied().unwrap_or("") + fn resolve(&self, ty: TypeId) -> &'a Type { + self.registry.resolve(ty).expect("in registry") + } + + fn ty_len(&self, data: &[u8], ty: TypeId) -> usize { + match self.resolve(ty).type_def() { + TypeDef::Primitive(p) => match p { + Primitive::U8 => mem::size_of::(), + Primitive::U16 => mem::size_of::(), + Primitive::U32 => mem::size_of::(), + Primitive::U64 => mem::size_of::(), + Primitive::U128 => mem::size_of::(), + Primitive::I8 => mem::size_of::(), + Primitive::I16 => mem::size_of::(), + Primitive::I32 => mem::size_of::(), + Primitive::I64 => mem::size_of::(), + Primitive::I128 => mem::size_of::(), + Primitive::Bool => mem::size_of::(), + Primitive::Char => mem::size_of::(), + Primitive::Str => { + let (l, p_size) = sequence_len(data); + l + p_size + } + _ => unimplemented!(), + }, + TypeDef::Composite(c) => c + .fields() + .iter() + .fold(0, |c, f| c + self.ty_len(&data[c..], f.ty().id())), + TypeDef::Variant(e) => { + let var = e + .variants() + .iter() + .find(|v| v.index() == data[0]) + .expect("variant"); + + if var.fields().is_empty() { + 1 // unit variant + } else { + var.fields() + .iter() + .fold(1, |c, f| c + self.ty_len(&data[c..], f.ty().id())) + } + } + TypeDef::Sequence(s) => { + let (len, prefix_size) = sequence_len(data); + let ty_id = s.type_param().id(); + (0..len).fold(prefix_size, |c, _| c + self.ty_len(&data[c..], ty_id)) + } + TypeDef::Array(a) => a.len().try_into().unwrap(), + TypeDef::Tuple(t) => t.fields().len(), + TypeDef::Compact(_) => compact_len(data), + TypeDef::BitSequence(_) => unimplemented!(), + } } } -impl Serialize for Value { +impl<'a> Serialize for Value<'a> { fn serialize(&self, ser: S) -> Result where S: serde::Serializer, { let mut data = self.data.clone(); + let ty = self.resolve(self.ty_id); use SerdeType::*; - match &self.ty { + match (ty, self.registry).into() { Bool => ser.serialize_bool(data.get_u8() != 0), U8 => ser.serialize_u8(data.get_u8()), U16 => ser.serialize_u16(data.get_u16_le()), @@ -73,7 +126,7 @@ impl Serialize for Value { let mut seq = ser.serialize_seq(Some(len))?; for _ in 0..len { - seq.serialize_element(&Value::chunk(&mut data, ty.clone()))?; + seq.serialize_element(&self.new_value(&mut data, ty))?; } seq.end() } @@ -83,8 +136,8 @@ impl Serialize for Value { let mut state = ser.serialize_map(Some(len))?; for _ in 0..len { - let key = Value::chunk(&mut data, ty_k.clone()); - let val = Value::chunk(&mut data, ty_v.clone()); + let key = self.new_value(&mut data, ty_k); + let val = self.new_value(&mut data, ty_v); state.serialize_entry(&key, &val)?; } state.end() @@ -92,28 +145,24 @@ impl Serialize for Value { Tuple(t) => { let mut state = ser.serialize_tuple(t.len())?; for i in 0..t.len() { - state.serialize_element(&Value::chunk(&mut data, t.type_info(i).clone()))?; + state.serialize_element(&self.new_value(&mut data, t.type_id(i)))?; } state.end() } Struct(fields) => { - let mut state = ser.serialize_struct(self.ty_name(), fields.len())?; - for f in fields { - state.serialize_field( - f.name().unwrap(), - &Value::chunk(&mut data, f.ty().type_info()), - )?; + let mut state = ser.serialize_map(Some(fields.len()))?; + for (name, ty) in fields { + state.serialize_key(&name)?; + state.serialize_value(&self.new_value(&mut data, ty))?; } state.end() } - StructUnit => ser.serialize_unit_struct(self.ty_name()), - StructNewType(ty) => { - ser.serialize_newtype_struct(self.ty_name(), &Value::chunk(&mut data, ty.clone())) - } + StructUnit => ser.serialize_unit(), + StructNewType(ty) => ser.serialize_newtype_struct("", &self.new_value(&mut data, ty)), StructTuple(fields) => { - let mut state = ser.serialize_tuple_struct(self.ty_name(), fields.len())?; - for f in fields { - state.serialize_field(&Value::chunk(&mut data, f.ty().type_info()))?; + let mut state = ser.serialize_tuple_struct("", fields.len())?; + for ty in fields { + state.serialize_field(&self.new_value(&mut data, ty))?; } state.end() } @@ -121,39 +170,38 @@ impl Serialize for Value { let variant = &ty.pick(data.get_u8()); match variant.into() { EnumVariant::OptionNone => ser.serialize_none(), - EnumVariant::OptionSome(ty) => ser.serialize_some(&Value::chunk(&mut data, ty)), - EnumVariant::Unit(idx, name) => { - ser.serialize_unit_variant(self.ty_name(), idx.into(), name) + EnumVariant::OptionSome(ty) => { + ser.serialize_some(&self.new_value(&mut data, ty)) } - EnumVariant::NewType(idx, name, ty) => ser.serialize_newtype_variant( - self.ty_name(), - idx.into(), - name, - &Value::chunk(&mut data, ty), - ), - - EnumVariant::Tuple(idx, name, fields) => { - let mut s = ser.serialize_tuple_variant( - self.ty_name(), - idx.into(), - name, - fields.len(), - )?; - for ty in fields { - s.serialize_field(&Value::chunk(&mut data, ty))?; - } + EnumVariant::Unit(_idx, name) => ser.serialize_str(name), + EnumVariant::NewType(_idx, name, ty) => { + let mut s = ser.serialize_map(Some(1))?; + s.serialize_key(name)?; + s.serialize_value(&self.new_value(&mut data, ty))?; s.end() } - EnumVariant::Struct(idx, name, fields) => { - let mut s = ser.serialize_struct_variant( - self.ty_name(), - idx.into(), - name, - fields.len(), + + EnumVariant::Tuple(_idx, name, fields) => { + let mut s = ser.serialize_map(Some(1))?; + s.serialize_key(name)?; + s.serialize_value( + &fields + .iter() + .map(|ty| self.new_value(&mut data, *ty)) + .collect::>(), )?; - for (fname, ty) in fields { - s.serialize_field(fname, &Value::chunk(&mut data, ty))?; - } + s.end() + } + EnumVariant::Struct(_idx, name, fields) => { + let mut s = ser.serialize_map(Some(1))?; + s.serialize_key(name)?; + s.serialize_value(&fields.iter().fold( + BTreeMap::new(), + |mut m, (name, ty)| { + m.insert(*name, self.new_value(&mut data, *ty)); + m + }, + ))?; s.end() } } @@ -162,58 +210,6 @@ impl Serialize for Value { } } -fn type_len(ty: &Type, data: &[u8]) -> usize { - match ty.type_def() { - TypeDef::Primitive(p) => match p { - Primitive::U8 => mem::size_of::(), - Primitive::U16 => mem::size_of::(), - Primitive::U32 => mem::size_of::(), - Primitive::U64 => mem::size_of::(), - Primitive::U128 => mem::size_of::(), - Primitive::I8 => mem::size_of::(), - Primitive::I16 => mem::size_of::(), - Primitive::I32 => mem::size_of::(), - Primitive::I64 => mem::size_of::(), - Primitive::I128 => mem::size_of::(), - Primitive::Bool => mem::size_of::(), - Primitive::Char => mem::size_of::(), - Primitive::Str => { - let (l, p_size) = sequence_len(data); - l + p_size - } - _ => unimplemented!(), - }, - TypeDef::Composite(c) => c - .fields() - .iter() - .fold(0, |c, f| c + type_len(&f.ty().type_info(), &data[c..])), - TypeDef::Variant(e) => { - let var = e - .variants() - .iter() - .find(|v| v.index() == data[0]) - .expect("variant"); - - if var.fields().is_empty() { - 1 // unit variant - } else { - var.fields() - .iter() - .fold(1, |c, f| c + type_len(&f.ty().type_info(), &data[c..])) - } - } - TypeDef::Sequence(s) => { - let (len, prefix_size) = sequence_len(data); - let ty = s.type_param().type_info(); - (0..len).fold(prefix_size, |c, _| c + type_len(&ty, &data[c..])) - } - TypeDef::Array(a) => a.len().try_into().unwrap(), - TypeDef::Tuple(t) => t.fields().len(), - TypeDef::Compact(_) => compact_len(data), - TypeDef::BitSequence(_) => unimplemented!(), - } -} - #[inline] fn compact_len(data: &[u8]) -> usize { match data[0] % 0b100 { @@ -251,88 +247,114 @@ mod tests { use anyhow::Error; use codec::Encode; use scale_info::{ + meta_type, prelude::{string::String, vec::Vec}, - TypeInfo, + Registry, TypeInfo, }; use serde_json::to_value; + fn register(_ty: &T) -> (u32, PortableRegistry) + where + T: TypeInfo + 'static, + { + let mut reg = Registry::new(); + let sym = reg.register_type(&meta_type::()); + (sym.id(), reg.into()) + } + #[test] fn serialize_u8() -> Result<(), Error> { - let extract_value = u8::MAX; - let data = extract_value.encode(); - let info = u8::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = u8::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_u16() -> Result<(), Error> { - let extract_value = u16::MAX; - let data = extract_value.encode(); - let info = u16::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = u16::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_u32() -> Result<(), Error> { - let extract_value = u32::MAX; - let data = extract_value.encode(); - let info = u32::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = u32::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_u64() -> Result<(), Error> { - let extract_value = u64::MAX; - let data = extract_value.encode(); - let info = u64::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = u64::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_i16() -> Result<(), Error> { - let extract_value = i16::MAX; - let data = extract_value.encode(); - let info = i16::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = i16::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_i32() -> Result<(), Error> { - let extract_value = i32::MAX; - let data = extract_value.encode(); - let info = i32::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = i32::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_i64() -> Result<(), Error> { - let extract_value = i64::MAX; - let data = extract_value.encode(); - let info = i64::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = i64::MAX; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_bool() -> Result<(), Error> { - let extract_value = true; - let data = extract_value.encode(); - let info = bool::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value = true; + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } @@ -342,52 +364,60 @@ mod tests { // let extract_value = '⚖'; // let data = extract_value.encode(); // let info = char::type_info(); - // let val = Value::new(data, info); + // let val = Value::new(data, info, reg); // assert_eq!(to_value(val)?, to_value(extract_value)?); // Ok(()) // } #[test] fn serialize_u8array() -> Result<(), Error> { - let extract_value: Vec = [2u8, u8::MAX].into(); - let data = extract_value.encode(); - let info = Vec::::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value: Vec = [2u8, u8::MAX].into(); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_u16array() -> Result<(), Error> { - let extract_value: Vec = [2u16, u16::MAX].into(); - let data = extract_value.encode(); - let info = Vec::::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value: Vec = [2u16, u16::MAX].into(); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_u32array() -> Result<(), Error> { - let extract_value: Vec = [2u32, u32::MAX].into(); - let data = extract_value.encode(); - let info = Vec::::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let in_value: Vec = [2u32, u32::MAX].into(); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_tuple() -> Result<(), Error> { - let extract_value: (i64, Vec, bool) = ( + let in_value: (i64, Vec, bool) = ( i64::MIN, vec!["hello".into(), "big".into(), "world".into()], true, ); - let data = extract_value.encode(); - let info = <(i64, Vec, bool)>::type_info(); - let val = Value::new(data, info); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } @@ -398,15 +428,16 @@ mod tests { bar: u32, baz: u32, } - let extract_value = Foo { + let in_value = Foo { bar: 123, baz: u32::MAX, }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(data, info); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); - assert_eq!(to_value(val)?, to_value(extract_value)?); + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } @@ -417,15 +448,16 @@ mod tests { bar: u8, baz: u8, } - let extract_value = Foo { + let in_value = Foo { bar: 123, baz: u8::MAX, }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(data, info); + let data = in_value.encode(); + let (id, reg) = register(&in_value); - assert_eq!(to_value(val)?, to_value(extract_value)?); + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } @@ -436,32 +468,34 @@ mod tests { bar: u64, baz: u64, } - let extract_value = Foo { + let in_value = Foo { bar: 123, baz: u64::MAX, }; - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(data, info); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); - assert_eq!(to_value(val)?, to_value(extract_value)?); + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } #[test] fn serialize_map() -> Result<(), Error> { - let value = { + let in_value = { let mut m = BTreeMap::::new(); m.insert("foo".into(), i32::MAX); m.insert("bar".into(), i32::MIN); m }; - let data = value.encode(); - let info = BTreeMap::::type_info(); - let val = Value::new(data, info); + let data = in_value.encode(); + let (id, reg) = register(&in_value); - assert_eq!(to_value(val)?, to_value(value)?); + let out_value = Value::new(data, id, ®); + + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } @@ -480,16 +514,17 @@ mod tests { baz: Option, lol: &'static [u8], } - let expected = Foo { + let in_value = Foo { bar: [Bar::That(i16::MAX), Bar::This].into(), baz: Some(Baz("aliquam malesuada bibendum arcu vitae".into())), lol: b"\0xFFsome stuff\0x00", }; - let data = expected.encode(); - let info = Foo::type_info(); - let out = Value::new(data, info); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); - assert_eq!(to_value(out)?, to_value(expected)?); + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } @@ -507,17 +542,18 @@ mod tests { B { bb: &'a str }, } - let extract_value = Foo( + let in_value = Foo( [1, 2, 3, 4], (false, None), Baz::A(Bar), Baz::B { bb: "lol" }, ); - let data = extract_value.encode(); - let info = Foo::type_info(); - let val = Value::new(data, info); + let data = in_value.encode(); + let (id, reg) = register(&in_value); + + let out_value = Value::new(data, id, ®); - assert_eq!(to_value(val)?, to_value(extract_value)?); + assert_eq!(to_value(out_value)?, to_value(in_value)?); Ok(()) } } From bf73fbc7657be1447b679ec504f3ee14faf2b554 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 15 Oct 2021 18:23:42 +0200 Subject: [PATCH 35/59] Adapt serializer to work with PortableRegistry --- Cargo.toml | 2 +- src/serializer.rs | 144 ++++++++++++++++++++++++++++------------------ 2 files changed, 89 insertions(+), 57 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3730c7e..8037061 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ scale-info = { version = "1.0.0", default-features = false } serde = { version = "1.0.126", default-features = false } [features] -default = ["std"] +default = ["std", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] diff --git a/src/serializer.rs b/src/serializer.rs index fc70bce..733927f 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,11 +1,12 @@ use crate::prelude::*; use bytes::BufMut; use core::fmt; -use scale_info::{Type, TypeInfo}; +use scale_info::{PortableRegistry, TypeInfo}; use serde::{ser, Serialize}; use crate::{EnumVariant, SerdeType, TupleOrArray}; +type TypeId = u32; type Result = core::result::Result; #[derive(TypeInfo)] @@ -22,12 +23,15 @@ where } #[inline] -pub fn to_vec_with_info(value: &T, info: impl Into>) -> Result> +pub fn to_vec_with_info( + value: &T, + registry_type: Option<(&PortableRegistry, TypeId)>, +) -> Result> where T: Serialize + ?Sized, { let mut out = vec![]; - to_bytes_with_info(&mut out, value, info)?; + to_bytes_with_info(&mut out, value, registry_type)?; Ok(out) } @@ -39,49 +43,57 @@ where to_bytes_with_info(bytes, value, None) } -pub fn to_bytes_with_info(bytes: B, value: &T, info: impl Into>) -> Result<()> +pub fn to_bytes_with_info( + bytes: B, + value: &T, + registry_type: Option<(&PortableRegistry, TypeId)>, +) -> Result<()> where T: Serialize + ?Sized, B: BufMut, { - let mut serializer = Serializer::new(bytes, info); + let mut serializer = Serializer::new(bytes, registry_type); value.serialize(&mut serializer)?; Ok(()) } /// A serializer that encodes types to SCALE with the option to coerce /// the output to an equivalent representation given by some type information. -pub struct Serializer { +pub struct Serializer<'reg, B> { out: B, ty: Option, + registry: Option<&'reg PortableRegistry>, } -impl Serializer +impl<'reg, B> Serializer<'reg, B> where B: BufMut, { - pub fn new(out: B, ty: impl Into>) -> Self { - Serializer { - out, - ty: ty.into().map(SerdeType::from), - } + pub fn new(out: B, registry_type: Option<(&'reg PortableRegistry, TypeId)>) -> Self { + let (registry, ty) = match registry_type + .map(|(reg, ty_id)| (reg, (reg.resolve(ty_id).unwrap(), reg).into())) + { + Some((reg, ty)) => (Some(reg), Some(ty)), + None => (None, None), + }; + Serializer { out, ty, registry } } } -impl<'a, B> ser::Serializer for &'a mut Serializer +impl<'a, 'reg, B> ser::Serializer for &'a mut Serializer<'reg, B> where B: BufMut, { type Ok = (); type Error = Error; - type SerializeSeq = TypedSerializer<'a, B>; - type SerializeTuple = TypedSerializer<'a, B>; - type SerializeTupleStruct = TypedSerializer<'a, B>; - type SerializeTupleVariant = TypedSerializer<'a, B>; - type SerializeMap = TypedSerializer<'a, B>; - type SerializeStruct = TypedSerializer<'a, B>; - type SerializeStructVariant = TypedSerializer<'a, B>; + type SerializeSeq = TypedSerializer<'a, 'reg, B>; + type SerializeTuple = TypedSerializer<'a, 'reg, B>; + type SerializeTupleStruct = TypedSerializer<'a, 'reg, B>; + type SerializeTupleVariant = TypedSerializer<'a, 'reg, B>; + type SerializeMap = TypedSerializer<'a, 'reg, B>; + type SerializeStruct = TypedSerializer<'a, 'reg, B>; + type SerializeStructVariant = TypedSerializer<'a, 'reg, B>; fn serialize_bool(self, v: bool) -> Result { self.maybe_some()?; @@ -301,7 +313,7 @@ where } } -impl<'a, B> Serializer +impl<'a, 'reg, B> Serializer<'reg, B> where B: BufMut, { @@ -310,30 +322,37 @@ where fn maybe_some(&mut self) -> Result<()> { match &self.ty { Some(SerdeType::Variant(ref name, v, _)) if name == "Option" => { - self.ty = v[1].fields().first().map(|f| f.ty().type_info().into()); + self.ty = v[1].fields().first().map(|f| self.resolve(f.ty().id())); self.out.put_u8(0x01); } _ => (), } Ok(()) } + + fn resolve(&self, ty_id: TypeId) -> SerdeType { + let reg = self.registry.expect("called heving type"); + let ty = reg.resolve(ty_id).expect("in registry"); + (ty, reg).into() + } } /// -pub enum TypedSerializer<'a, B> { - Empty(&'a mut Serializer), - Composite(&'a mut Serializer, Vec), - Sequence(&'a mut Serializer, Type), - Enum(&'a mut Serializer), +pub enum TypedSerializer<'a, 'reg, B> { + Empty(&'a mut Serializer<'reg, B>), + Composite(&'a mut Serializer<'reg, B>, Vec), + Sequence(&'a mut Serializer<'reg, B>, TypeId), + Enum(&'a mut Serializer<'reg, B>), } -impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { - fn from(ser: &'a mut Serializer) -> Self { +impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, 'reg, B> { + fn from(ser: &'a mut Serializer<'reg, B>) -> Self { use SerdeType::*; match ser.ty.take() { - Some(Struct(fields) | StructTuple(fields)) => { - Self::Composite(ser, fields.iter().map(|f| f.ty().type_info()).collect()) + Some(Struct(fields)) => { + Self::Composite(ser, fields.iter().map(|(_, ty)| *ty).collect()) } + Some(StructTuple(fields)) => Self::Composite(ser, fields), Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty), Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Composite(ser, fields), Some(Sequence(ty)) => Self::Sequence(ser, ty), @@ -354,8 +373,8 @@ impl<'a, B: 'a> From<&'a mut Serializer> for TypedSerializer<'a, B> { } } -impl<'a, B> TypedSerializer<'a, B> { - fn serializer(&mut self) -> &mut Serializer { +impl<'a, 'reg, B> TypedSerializer<'a, 'reg, B> { + fn serializer(&mut self) -> &mut Serializer<'reg, B> { match self { Self::Empty(ser) | Self::Composite(ser, _) @@ -365,7 +384,7 @@ impl<'a, B> TypedSerializer<'a, B> { } } -impl<'a, B> ser::SerializeMap for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeMap for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -398,17 +417,17 @@ where { match self { TypedSerializer::Composite(ser, types) => { - let mut ty = types.remove(0).into(); + let mut ty = ser.resolve(types.remove(0)); // serde_json unwraps newtypes - if let SerdeType::StructNewType(t) = ty { - ty = t.into() + if let SerdeType::StructNewType(ty_id) = ty { + ty = ser.resolve(ty_id) } ser.ty = Some(ty); } TypedSerializer::Enum(ser) => { if let Some(var @ SerdeType::Variant(_, _, Some(_))) = &ser.ty { - if let EnumVariant::NewType(_, _, ty) = var.into() { - ser.ty = Some(ty.into()); + if let EnumVariant::NewType(_, _, ty_id) = var.into() { + ser.ty = Some(ser.resolve(ty_id)); } } } @@ -422,7 +441,7 @@ where } } -impl<'a, B> ser::SerializeSeq for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeSeq for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -435,16 +454,17 @@ where { match self { TypedSerializer::Composite(ser, types) => { - let mut ty = types.remove(0).into(); - if let SerdeType::StructNewType(t) = ty { - ty = t.into(); + let mut ty = ser.resolve(types.remove(0)); + if let SerdeType::StructNewType(ty_id) = ty { + ty = ser.resolve(ty_id); } ser.ty = Some(ty); } - TypedSerializer::Sequence(ser, ty) => { - ser.ty = Some(match ty.into() { - SerdeType::StructNewType(t) => t.into(), - _ => ty.into(), + TypedSerializer::Sequence(ser, ty_id) => { + let ty = ser.resolve(*ty_id); + ser.ty = Some(match ty { + SerdeType::StructNewType(ty_id) => ser.resolve(ty_id), + _ => ty, }); } _ => {} @@ -457,7 +477,7 @@ where } } -impl<'a, B> ser::SerializeStruct for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeStruct for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -476,7 +496,7 @@ where } } -impl<'a, B> ser::SerializeStructVariant for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeStructVariant for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -495,7 +515,7 @@ where } } -impl<'a, B> ser::SerializeTuple for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeTuple for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -514,7 +534,7 @@ where } } -impl<'a, B> ser::SerializeTupleStruct for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeTupleStruct for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -533,7 +553,7 @@ where } } -impl<'a, B> ser::SerializeTupleVariant for TypedSerializer<'a, B> +impl<'a, 'reg, B> ser::SerializeTupleVariant for TypedSerializer<'a, 'reg, B> where B: BufMut, { @@ -608,7 +628,7 @@ mod tests { use super::*; use codec::Encode; use core::mem::size_of; - use scale_info::TypeInfo; + use scale_info::{meta_type, Registry, TypeInfo}; use serde_json::to_value; #[test] @@ -855,6 +875,15 @@ mod tests { Ok(()) } + fn register(_ty: &T) -> (TypeId, PortableRegistry) + where + T: TypeInfo + 'static, + { + let mut reg = Registry::new(); + let sym = reg.register_type(&meta_type::()); + (sym.id(), reg.into()) + } + #[test] fn json_simple() -> Result<()> { #[derive(Debug, Serialize, Encode, TypeInfo)] @@ -873,9 +902,10 @@ mod tests { }; let mut out = Vec::::new(); let expected = input.encode(); + let (id, reg) = register(&input); let json_input = to_value(&input).unwrap(); - to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; + to_bytes_with_info(&mut out, &json_input, Some((®, id)))?; assert_eq!(out, expected); Ok(()) @@ -915,9 +945,10 @@ mod tests { }; let mut out = Vec::::new(); let expected = input.encode(); + let (id, reg) = register(&input); let json_input = to_value(&input).unwrap(); - to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; + to_bytes_with_info(&mut out, &json_input, Some((®, id)))?; assert_eq!(out, expected); Ok(()) @@ -945,9 +976,10 @@ mod tests { }; let mut out = Vec::::new(); let expected = input.encode(); + let (id, reg) = register(&input); let json_input = to_value(&input).unwrap(); - to_bytes_with_info(&mut out, &json_input, Foo::type_info())?; + to_bytes_with_info(&mut out, &json_input, Some((®, id)))?; assert_eq!(out, expected); Ok(()) From a3e45330eeb2b2b4533247d1df5e1b5c5d8ded7a Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 15 Oct 2021 18:39:53 +0200 Subject: [PATCH 36/59] Fix WASM and no_std compilation --- Cargo.toml | 2 +- src/lib.rs | 17 +++++++++-------- src/value.rs | 5 ++--- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 8037061..da1c691 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.5.3" +version = "0.6.0" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" diff --git a/src/lib.rs b/src/lib.rs index 330d047..ce0d8bd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -23,6 +23,7 @@ mod prelude { string::{String, ToString}, vec::Vec, }; + pub use core::ops::Deref; } type Type = scale_info::Type; @@ -96,7 +97,7 @@ impl From<(&Type, &PortableRegistry)> for SerdeType { .segments() .last() .cloned() - .unwrap_or_else(String::new); + .unwrap_or_else(|| "".into()); match ty.type_def() { Def::Composite(c) => { @@ -114,7 +115,7 @@ impl From<(&Type, &PortableRegistry)> for SerdeType { Self::Struct( fields .iter() - .map(|f| (f.name().unwrap().into(), f.ty().id())) + .map(|f| (f.name().unwrap().deref().into(), f.ty().id())) .collect(), ) } @@ -218,13 +219,13 @@ impl<'a> From<&'a SerdeType> for EnumVariant<'a> { SerdeType::Variant(name, variants, Some(idx)) => { let variant = variants.first().expect("single variant"); let fields = variant.fields(); - let vname = &*variant.name(); + let vname = variant.name().as_ref(); if fields.is_empty() { if name == "Option" && vname == "None" { Self::OptionNone } else { - Self::Unit(*idx, vname.as_str()) + Self::Unit(*idx, &vname) } } else if is_tuple!(variant) { if fields.len() == 1 { @@ -232,18 +233,18 @@ impl<'a> From<&'a SerdeType> for EnumVariant<'a> { return if name == "Option" && variant.name() == &"Some" { Self::OptionSome(ty) } else { - Self::NewType(*idx, vname.as_str(), ty) + Self::NewType(*idx, &vname, ty) }; } else { let fields = fields.iter().map(|f| f.ty().id()).collect(); - Self::Tuple(*idx, vname.as_str(), fields) + Self::Tuple(*idx, &vname, fields) } } else { let fields = fields .iter() - .map(|f| (f.name().unwrap().as_str(), f.ty().id())) + .map(|f| (f.name().unwrap().deref(), f.ty().id())) .collect(); - Self::Struct(*idx, vname.as_str(), fields) + Self::Struct(*idx, &vname, fields) } } _ => panic!("Only for enum variants"), diff --git a/src/value.rs b/src/value.rs index cb37a0f..0ce4d11 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,8 +1,7 @@ use crate::{EnumVariant, SerdeType}; -use alloc::collections::BTreeMap; +use alloc::{collections::BTreeMap, vec::Vec}; use bytes::{Buf, Bytes}; -use core::convert::TryInto; -use core::str; +use core::{convert::TryInto, str}; use scale_info::{prelude::*, PortableRegistry, TypeDefPrimitive as Primitive}; use serde::ser::{SerializeMap, SerializeSeq, SerializeTuple, SerializeTupleStruct}; use serde::Serialize; From 7d921338eec7998b5876a9938ab97cd0ce989b7c Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 15 Oct 2021 20:43:14 +0200 Subject: [PATCH 37/59] Implement Display for Value with JSON feature Override Debug to be less verbose Impl AsRef<[u8]> --- Cargo.toml | 6 ++++-- src/lib.rs | 2 +- src/value.rs | 45 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index da1c691..30efeb9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.6.0" +version = "0.7.1" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" @@ -11,11 +11,13 @@ license = "Apache-2.0" bytes = { version = "1.1.0", default-features = false } scale-info = { version = "1.0.0", default-features = false } serde = { version = "1.0.126", default-features = false } +serde_json = { version = "1.0.66", default-features = false, optional = true } [features] -default = ["std", "experimental-serializer"] +default = ["std", "json", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] +json = ["serde_json"] [dev-dependencies] anyhow = "1.0.42" diff --git a/src/lib.rs b/src/lib.rs index ce0d8bd..8b082c6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -107,7 +107,7 @@ impl From<(&Type, &PortableRegistry)> for SerdeType { } else if is_map(ty) { let (k, v) = map_types(c); Self::Map(k, v) - } else if fields.len() == 1 { + } else if fields.len() == 1 && fields.first().unwrap().name().is_none() { Self::StructNewType(fields.first().unwrap().ty().id()) } else if is_tuple!(c) { Self::StructTuple(fields.iter().map(|f| f.ty().id()).collect()) diff --git a/src/value.rs b/src/value.rs index 0ce4d11..7fc1145 100644 --- a/src/value.rs +++ b/src/value.rs @@ -12,7 +12,6 @@ type TypeDef = scale_info::TypeDef; /// A container for SCALE encoded data that can serialize types directly /// with the help of a type registry and without using an intermediate representation. -#[derive(Debug)] pub struct Value<'a> { data: Bytes, ty_id: TypeId, @@ -238,6 +237,35 @@ fn sequence_len(data: &[u8]) -> (usize, usize) { ) } +impl<'reg> AsRef<[u8]> for Value<'reg> { + fn as_ref(&self) -> &[u8] { + self.data.as_ref() + } +} + +impl<'reg> core::fmt::Debug for Value<'reg> { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!( + f, + "Value {{ data: {:?}, type({}): {:?} }}", + self.data, + self.ty_id, + self.registry.resolve(self.ty_id).unwrap().type_def() + ) + } +} + +#[cfg(feature = "json")] +impl<'reg> core::fmt::Display for Value<'reg> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + serde_json::to_string(self).map_err(|_| fmt::Error)? + ) + } +} + #[cfg(test)] mod tests { use alloc::collections::BTreeMap; @@ -261,6 +289,21 @@ mod tests { (sym.id(), reg.into()) } + #[test] + fn display_as_json() { + #[derive(Encode, TypeInfo)] + struct Foo { + bar: String, + } + let in_value = Foo { bar: "BAZ".into() }; + + let data = in_value.encode(); + let (id, reg) = register(&in_value); + let out_value = Value::new(data, id, ®).to_string(); + + assert_eq!("{\"bar\":\"BAZ\"}", out_value); + } + #[test] fn serialize_u8() -> Result<(), Error> { let in_value = u8::MAX; From de5683d10b55a0784ad52ee1c31aa4b07700f310 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 15 Oct 2021 21:29:53 +0200 Subject: [PATCH 38/59] Convert Value to JSON --- Cargo.toml | 2 +- src/lib.rs | 2 ++ src/value.rs | 7 +++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 30efeb9..1c8cf9f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.7.1" +version = "0.7.2" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" diff --git a/src/lib.rs b/src/lib.rs index 8b082c6..8b6f13e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,8 @@ extern crate alloc; mod serializer; mod value; +#[cfg(feature = "json")] +pub use serde_json::Value as JsonValue; #[cfg(feature = "experimental-serializer")] pub use serializer::{to_bytes, to_bytes_with_info, to_vec, to_vec_with_info, Serializer}; pub use value::Value; diff --git a/src/value.rs b/src/value.rs index 7fc1145..c74f764 100644 --- a/src/value.rs +++ b/src/value.rs @@ -266,6 +266,13 @@ impl<'reg> core::fmt::Display for Value<'reg> { } } +#[cfg(feature = "json")] +impl<'reg> Into for Value<'reg> { + fn into(self) -> crate::JsonValue { + serde_json::value::to_value(self).unwrap() + } +} + #[cfg(test)] mod tests { use alloc::collections::BTreeMap; From 0ae21da2568fcd05612edae0193d0dd9cabe88ab Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 20 Oct 2021 16:48:11 +0200 Subject: [PATCH 39/59] Make Value Encodable --- Cargo.toml | 17 +++++++++-------- src/value.rs | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 1c8cf9f..422f437 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.7.2" +version = "0.7.3" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" @@ -10,18 +10,19 @@ license = "Apache-2.0" [dependencies] bytes = { version = "1.1.0", default-features = false } scale-info = { version = "1.0.0", default-features = false } -serde = { version = "1.0.126", default-features = false } -serde_json = { version = "1.0.66", default-features = false, optional = true } +serde = { version = "1.0.130", default-features = false } +serde_json = { version = "1.0.68", default-features = false, optional = true } +codec = { version = "2.3.1", package = "parity-scale-codec", default-features = false, optional = true } [features] -default = ["std", "json", "experimental-serializer"] +default = ["std", "codec", "json", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] json = ["serde_json"] [dev-dependencies] -anyhow = "1.0.42" -codec = { version = "2.2.0", package = "parity-scale-codec", features = ["derive"] } +anyhow = "1.0.44" +codec = { version = "2.3.1", package = "parity-scale-codec", features = ["derive"] } scale-info = { version = "1.0.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.126", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.66", default-features = false, features = ["alloc"] } +serde = { version = "1.0.130", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.68", default-features = false, features = ["alloc"] } diff --git a/src/value.rs b/src/value.rs index c74f764..418de97 100644 --- a/src/value.rs +++ b/src/value.rs @@ -243,6 +243,15 @@ impl<'reg> AsRef<[u8]> for Value<'reg> { } } +impl<'reg> codec::Encode for Value<'reg> { + fn size_hint(&self) -> usize { + self.data.len() + } + fn using_encoded R>(&self, f: F) -> R { + f(self.data.as_ref()) + } +} + impl<'reg> core::fmt::Debug for Value<'reg> { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { write!( @@ -311,6 +320,16 @@ mod tests { assert_eq!("{\"bar\":\"BAZ\"}", out_value); } + #[test] + fn encodable() { + let input = u8::MAX; + let (ty, reg) = register(&input); + let value = Value::new(b"1234".as_ref(), ty, ®); + + let expected: &[u8] = value.as_ref(); + assert_eq!(value.encode(), expected); + } + #[test] fn serialize_u8() -> Result<(), Error> { let in_value = u8::MAX; From 9b2425c3c41ad48b64761aacd8fc7700414c6eff Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 20 Oct 2021 17:01:33 +0200 Subject: [PATCH 40/59] Allow tests to run with no-default-features --- Cargo.toml | 2 +- src/value.rs | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 422f437..00bd6be 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.7.3" +version = "0.7.4" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" diff --git a/src/value.rs b/src/value.rs index 418de97..850b144 100644 --- a/src/value.rs +++ b/src/value.rs @@ -243,6 +243,7 @@ impl<'reg> AsRef<[u8]> for Value<'reg> { } } +#[cfg(feature = "codec")] impl<'reg> codec::Encode for Value<'reg> { fn size_hint(&self) -> usize { self.data.len() @@ -305,6 +306,7 @@ mod tests { (sym.id(), reg.into()) } + #[cfg(feature = "json")] #[test] fn display_as_json() { #[derive(Encode, TypeInfo)] @@ -320,6 +322,7 @@ mod tests { assert_eq!("{\"bar\":\"BAZ\"}", out_value); } + #[cfg(feature = "codec")] #[test] fn encodable() { let input = u8::MAX; From 69b4d62134feb088ee12d512750a261bcd2edc7d Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 25 Oct 2021 16:12:48 +0200 Subject: [PATCH 41/59] Expose Bytes --- Cargo.toml | 2 +- src/lib.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 00bd6be..24c19e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.7.4" +version = "0.7.5" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" diff --git a/src/lib.rs b/src/lib.rs index 8b6f13e..d17b005 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,7 @@ extern crate alloc; mod serializer; mod value; +pub use bytes::Bytes; #[cfg(feature = "json")] pub use serde_json::Value as JsonValue; #[cfg(feature = "experimental-serializer")] From e8f1d1d44ae46851f552c00cd513ab3582d3f20d Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 28 Oct 2021 12:41:46 +0200 Subject: [PATCH 42/59] Rename SpecificType and impl serialize/encode --- Cargo.toml | 4 ++-- src/lib.rs | 40 +++++++++++++++------------------------- src/serializer.rs | 44 ++++++++++++++++++++++---------------------- src/value.rs | 4 ++-- 4 files changed, 41 insertions(+), 51 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 24c19e3..68438a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.7.5" +version = "0.8.0" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" @@ -9,7 +9,7 @@ license = "Apache-2.0" [dependencies] bytes = { version = "1.1.0", default-features = false } -scale-info = { version = "1.0.0", default-features = false } +scale-info = { version = "1.0.0", default-features = false, features = ["serde"] } serde = { version = "1.0.130", default-features = false } serde_json = { version = "1.0.68", default-features = false, optional = true } codec = { version = "2.3.1", package = "parity-scale-codec", default-features = false, optional = true } diff --git a/src/lib.rs b/src/lib.rs index d17b005..2262e03 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -43,8 +43,9 @@ macro_rules! is_tuple { /// A convenient representation of the scale-info types to a format /// that matches serde model more closely #[rustfmt::skip] -#[derive(Debug, Clone)] -pub enum SerdeType { +#[derive(Debug, Clone, serde::Serialize)] +#[cfg_attr(feature = "codec", derive(codec::Encode))] +pub enum SpecificType { Bool, U8, U16, U32, U64, U128, I8, I16, I32, I64, I128, @@ -58,19 +59,7 @@ pub enum SerdeType { Variant(String, Vec, Option), } -// impl From<&mut Type> for SerdeType { -// fn from(ty: &mut Type) -> Self { -// ty.clone().into() -// } -// } - -// impl From<&Type> for SerdeType { -// fn from(ty: &Type) -> Self { -// ty.clone().into() -// } -// } - -impl From<(&Type, &PortableRegistry)> for SerdeType { +impl From<(&Type, &PortableRegistry)> for SpecificType { fn from((ty, registry): (&Type, &PortableRegistry)) -> Self { use scale_info::{TypeDef, TypeDefComposite, TypeDefPrimitive}; type Def = TypeDef; @@ -160,13 +149,13 @@ impl From<(&Type, &PortableRegistry)> for SerdeType { } // Utilities for enum variants -impl SerdeType { +impl SpecificType { fn pick(&self, index: u8) -> Self { match self { - SerdeType::Variant(name, variant, Some(_)) => { + SpecificType::Variant(name, variant, Some(_)) => { Self::Variant(name.to_string(), variant.to_vec(), Some(index)) } - SerdeType::Variant(name, variants, None) => { + SpecificType::Variant(name, variants, None) => { let v = variants.iter().find(|v| v.index() == index).unwrap(); Self::Variant(name.clone(), vec![v.clone()], Some(index)) } @@ -182,8 +171,8 @@ impl SerdeType { B: AsRef<[u8]> + PartialEq + core::fmt::Debug, { match self { - SerdeType::Variant(_, _, Some(_)) => self, - SerdeType::Variant(_, ref mut variants, idx @ None) => { + SpecificType::Variant(_, _, Some(_)) => self, + SpecificType::Variant(_, ref mut variants, idx @ None) => { let i = variants .iter() .map(|v| get_field(v)) @@ -200,7 +189,7 @@ impl SerdeType { #[cfg(feature = "experimental-serializer")] fn variant_id(&self) -> u8 { match self { - SerdeType::Variant(_, _, Some(id)) => *id, + SpecificType::Variant(_, _, Some(id)) => *id, _ => panic!("Only for enum variants"), } } @@ -216,10 +205,10 @@ enum EnumVariant<'a> { Struct(u8, &'a str, Vec<(&'a str, TypeId)>), } -impl<'a> From<&'a SerdeType> for EnumVariant<'a> { - fn from(ty: &'a SerdeType) -> Self { +impl<'a> From<&'a SpecificType> for EnumVariant<'a> { + fn from(ty: &'a SpecificType) -> Self { match ty { - SerdeType::Variant(name, variants, Some(idx)) => { + SpecificType::Variant(name, variants, Some(idx)) => { let variant = variants.first().expect("single variant"); let fields = variant.fields(); let vname = variant.name().as_ref(); @@ -255,7 +244,8 @@ impl<'a> From<&'a SerdeType> for EnumVariant<'a> { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, serde::Serialize)] +#[cfg_attr(feature = "codec", derive(codec::Encode))] pub enum TupleOrArray { Array(TypeId, u32), Tuple(Vec), diff --git a/src/serializer.rs b/src/serializer.rs index 733927f..921c73f 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -4,7 +4,7 @@ use core::fmt; use scale_info::{PortableRegistry, TypeInfo}; use serde::{ser, Serialize}; -use crate::{EnumVariant, SerdeType, TupleOrArray}; +use crate::{EnumVariant, SpecificType, TupleOrArray}; type TypeId = u32; type Result = core::result::Result; @@ -61,7 +61,7 @@ where /// the output to an equivalent representation given by some type information. pub struct Serializer<'reg, B> { out: B, - ty: Option, + ty: Option, registry: Option<&'reg PortableRegistry>, } @@ -121,9 +121,9 @@ where fn serialize_i64(self, v: i64) -> Result { match self.ty { - Some(SerdeType::I8) => self.serialize_i8(v as i8)?, - Some(SerdeType::I16) => self.serialize_i16(v as i16)?, - Some(SerdeType::I32) => self.serialize_i32(v as i32)?, + Some(SpecificType::I8) => self.serialize_i8(v as i8)?, + Some(SpecificType::I16) => self.serialize_i16(v as i16)?, + Some(SpecificType::I32) => self.serialize_i32(v as i32)?, _ => { self.maybe_some()?; self.out.put_i64_le(v) @@ -153,12 +153,12 @@ where fn serialize_u64(self, v: u64) -> Result { // all numbers in serde_json are the same match self.ty { - Some(SerdeType::I8) => self.serialize_i8(v as i8)?, - Some(SerdeType::I16) => self.serialize_i16(v as i16)?, - Some(SerdeType::I32) => self.serialize_i32(v as i32)?, - Some(SerdeType::U8) => self.serialize_u8(v as u8)?, - Some(SerdeType::U16) => self.serialize_u16(v as u16)?, - Some(SerdeType::U32) => self.serialize_u32(v as u32)?, + Some(SpecificType::I8) => self.serialize_i8(v as i8)?, + Some(SpecificType::I16) => self.serialize_i16(v as i16)?, + Some(SpecificType::I32) => self.serialize_i32(v as i32)?, + Some(SpecificType::U8) => self.serialize_u8(v as u8)?, + Some(SpecificType::U16) => self.serialize_u16(v as u16)?, + Some(SpecificType::U32) => self.serialize_u32(v as u32)?, _ => { self.maybe_some()?; self.out.put_u64_le(v); @@ -181,7 +181,7 @@ where fn serialize_str(self, v: &str) -> Result { self.maybe_some()?; - if let Some(ref mut var @ SerdeType::Variant(_, _, None)) = &mut self.ty { + if let Some(ref mut var @ SpecificType::Variant(_, _, None)) = &mut self.ty { var.pick_mut(to_vec(v)?, |k| to_vec(k.name()).unwrap()); self.out.put_u8(var.variant_id()); return Ok(()); @@ -255,7 +255,7 @@ where fn serialize_seq(self, len: Option) -> Result { self.maybe_some()?; - if matches!(self.ty, None | Some(SerdeType::Sequence(_))) { + if matches!(self.ty, None | Some(SpecificType::Sequence(_))) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -289,7 +289,7 @@ where fn serialize_map(self, len: Option) -> Result { self.maybe_some()?; - if matches!(self.ty, None | Some(SerdeType::Map(_, _))) { + if matches!(self.ty, None | Some(SpecificType::Map(_, _))) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -321,7 +321,7 @@ where // if the type info says its an Option assume its Some and extract the inner type fn maybe_some(&mut self) -> Result<()> { match &self.ty { - Some(SerdeType::Variant(ref name, v, _)) if name == "Option" => { + Some(SpecificType::Variant(ref name, v, _)) if name == "Option" => { self.ty = v[1].fields().first().map(|f| self.resolve(f.ty().id())); self.out.put_u8(0x01); } @@ -330,7 +330,7 @@ where Ok(()) } - fn resolve(&self, ty_id: TypeId) -> SerdeType { + fn resolve(&self, ty_id: TypeId) -> SpecificType { let reg = self.registry.expect("called heving type"); let ty = reg.resolve(ty_id).expect("in registry"); (ty, reg).into() @@ -347,7 +347,7 @@ pub enum TypedSerializer<'a, 'reg, B> { impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, 'reg, B> { fn from(ser: &'a mut Serializer<'reg, B>) -> Self { - use SerdeType::*; + use SpecificType::*; match ser.ty.take() { Some(Struct(fields)) => { Self::Composite(ser, fields.iter().map(|(_, ty)| *ty).collect()) @@ -397,7 +397,7 @@ where { match self { TypedSerializer::Enum(ser) => { - if let Some(ref mut var @ SerdeType::Variant(_, _, None)) = ser.ty { + if let Some(ref mut var @ SpecificType::Variant(_, _, None)) = ser.ty { let key_data = to_vec(key)?; // assume the key is the name of the variant var.pick_mut(key_data, |v| to_vec(v.name()).unwrap()) @@ -419,13 +419,13 @@ where TypedSerializer::Composite(ser, types) => { let mut ty = ser.resolve(types.remove(0)); // serde_json unwraps newtypes - if let SerdeType::StructNewType(ty_id) = ty { + if let SpecificType::StructNewType(ty_id) = ty { ty = ser.resolve(ty_id) } ser.ty = Some(ty); } TypedSerializer::Enum(ser) => { - if let Some(var @ SerdeType::Variant(_, _, Some(_))) = &ser.ty { + if let Some(var @ SpecificType::Variant(_, _, Some(_))) = &ser.ty { if let EnumVariant::NewType(_, _, ty_id) = var.into() { ser.ty = Some(ser.resolve(ty_id)); } @@ -455,7 +455,7 @@ where match self { TypedSerializer::Composite(ser, types) => { let mut ty = ser.resolve(types.remove(0)); - if let SerdeType::StructNewType(ty_id) = ty { + if let SpecificType::StructNewType(ty_id) = ty { ty = ser.resolve(ty_id); } ser.ty = Some(ty); @@ -463,7 +463,7 @@ where TypedSerializer::Sequence(ser, ty_id) => { let ty = ser.resolve(*ty_id); ser.ty = Some(match ty { - SerdeType::StructNewType(ty_id) => ser.resolve(ty_id), + SpecificType::StructNewType(ty_id) => ser.resolve(ty_id), _ => ty, }); } diff --git a/src/value.rs b/src/value.rs index 850b144..6949496 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,4 +1,4 @@ -use crate::{EnumVariant, SerdeType}; +use crate::{EnumVariant, SpecificType}; use alloc::{collections::BTreeMap, vec::Vec}; use bytes::{Buf, Bytes}; use core::{convert::TryInto, str}; @@ -98,7 +98,7 @@ impl<'a> Serialize for Value<'a> { let mut data = self.data.clone(); let ty = self.resolve(self.ty_id); - use SerdeType::*; + use SpecificType::*; match (ty, self.registry).into() { Bool => ser.serialize_bool(data.get_u8() != 0), U8 => ser.serialize_u8(data.get_u8()), From 150ba73b8a9bb9f3ed94beafd8e74ee23ee72698 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Sat, 4 Dec 2021 07:14:55 +0100 Subject: [PATCH 43/59] Serialize from unordered list of fields (#5) --- Cargo.toml | 8 +-- src/lib.rs | 11 ++-- src/serializer.rs | 138 +++++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 139 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 68438a6..49438f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,18 +11,18 @@ license = "Apache-2.0" bytes = { version = "1.1.0", default-features = false } scale-info = { version = "1.0.0", default-features = false, features = ["serde"] } serde = { version = "1.0.130", default-features = false } -serde_json = { version = "1.0.68", default-features = false, optional = true } +serde_json = { version = "1.0.72", default-features = false, optional = true } codec = { version = "2.3.1", package = "parity-scale-codec", default-features = false, optional = true } [features] default = ["std", "codec", "json", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] -json = ["serde_json"] +json = ["serde_json/preserve_order"] [dev-dependencies] -anyhow = "1.0.44" +anyhow = "1.0.51" codec = { version = "2.3.1", package = "parity-scale-codec", features = ["derive"] } scale-info = { version = "1.0.0", default-features = false, features = ["derive"] } serde = { version = "1.0.130", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.68", default-features = false, features = ["alloc"] } +serde_json = { version = "1.0.72", default-features = false, features = ["alloc"] } diff --git a/src/lib.rs b/src/lib.rs index 2262e03..c13edaa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,8 @@ pub use bytes::Bytes; pub use serde_json::Value as JsonValue; #[cfg(feature = "experimental-serializer")] pub use serializer::{to_bytes, to_bytes_with_info, to_vec, to_vec_with_info, Serializer}; +#[cfg(feature = "json")] +pub use serializer::{to_bytes_from_iter, to_vec_from_iter}; pub use value::Value; use prelude::*; @@ -164,23 +166,22 @@ impl SpecificType { } #[cfg(feature = "experimental-serializer")] - fn pick_mut(&mut self, selection: A, get_field: F) -> &Self + fn pick_mut(&mut self, selection: A, get_field: F) -> Option<&Self> where F: Fn(&Variant) -> B, A: AsRef<[u8]> + PartialEq + core::fmt::Debug, B: AsRef<[u8]> + PartialEq + core::fmt::Debug, { match self { - SpecificType::Variant(_, _, Some(_)) => self, + SpecificType::Variant(_, _, Some(_)) => Some(self), SpecificType::Variant(_, ref mut variants, idx @ None) => { let i = variants .iter() .map(|v| get_field(v)) - .position(|f| f.as_ref() == selection.as_ref()) - .expect("index") as u8; + .position(|f| f.as_ref() == selection.as_ref())? as u8; variants.retain(|v| v.index() == i); *idx = Some(i); - self + Some(self) } _ => panic!("Only for enum variants"), } diff --git a/src/serializer.rs b/src/serializer.rs index 921c73f..8ba2dcd 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -57,6 +57,58 @@ where Ok(()) } +#[cfg(feature = "json")] +pub fn to_bytes_from_iter( + bytes: B, + iter: I, + registry_type: (&PortableRegistry, TypeId), +) -> Result<()> +where + B: BufMut, + I: IntoIterator, + K: Into, + V: Into, +{ + let ty = registry_type + .0 + .resolve(registry_type.1) + .ok_or_else(|| Error::BadInput("Type not in registry".into()))?; + let obj = iter.into_iter().collect::(); + let val: crate::JsonValue = if let scale_info::TypeDef::Composite(ty) = ty.type_def() { + ty.fields() + .iter() + .map(|f| { + let name = f.name().expect("named field"); + Ok(( + name.deref(), + obj.get(name) + .ok_or_else(|| Error::BadInput(format!("missing field {}", name)))? + .clone(), + )) + }) + .collect::>()? + } else { + return Err(Error::Type(ty.clone())); + }; + + to_bytes_with_info(bytes, &val, Some(registry_type)) +} + +#[cfg(feature = "json")] +pub fn to_vec_from_iter( + iter: I, + registry_type: (&PortableRegistry, TypeId), +) -> Result> +where + I: IntoIterator, + K: Into, + V: Into, +{ + let mut out = vec![]; + to_bytes_from_iter(&mut out, iter, registry_type)?; + Ok(out) +} + /// A serializer that encodes types to SCALE with the option to coerce /// the output to an equivalent representation given by some type information. pub struct Serializer<'reg, B> { @@ -70,9 +122,12 @@ where B: BufMut, { pub fn new(out: B, registry_type: Option<(&'reg PortableRegistry, TypeId)>) -> Self { - let (registry, ty) = match registry_type - .map(|(reg, ty_id)| (reg, (reg.resolve(ty_id).unwrap(), reg).into())) - { + let (registry, ty) = match registry_type.map(|(reg, ty_id)| { + ( + reg, + (reg.resolve(ty_id).expect("exists in registry"), reg).into(), + ) + }) { Some((reg, ty)) => (Some(reg), Some(ty)), None => (None, None), }; @@ -152,6 +207,7 @@ where fn serialize_u64(self, v: u64) -> Result { // all numbers in serde_json are the same + self.maybe_some()?; match self.ty { Some(SpecificType::I8) => self.serialize_i8(v as i8)?, Some(SpecificType::I16) => self.serialize_i16(v as i16)?, @@ -160,7 +216,6 @@ where Some(SpecificType::U16) => self.serialize_u16(v as u16)?, Some(SpecificType::U32) => self.serialize_u32(v as u32)?, _ => { - self.maybe_some()?; self.out.put_u64_le(v); } } @@ -181,11 +236,10 @@ where fn serialize_str(self, v: &str) -> Result { self.maybe_some()?; - if let Some(ref mut var @ SpecificType::Variant(_, _, None)) = &mut self.ty { - var.pick_mut(to_vec(v)?, |k| to_vec(k.name()).unwrap()); - self.out.put_u8(var.variant_id()); + if self.maybe_other(v)?.is_some() { return Ok(()); } + compact_number(v.len(), &mut self.out); self.out.put(v.as_bytes()); Ok(()) @@ -331,10 +385,31 @@ where } fn resolve(&self, ty_id: TypeId) -> SpecificType { - let reg = self.registry.expect("called heving type"); + let reg = self.registry.expect("called having type"); let ty = reg.resolve(ty_id).expect("in registry"); (ty, reg).into() } + + #[inline] + fn maybe_other(&mut self, val: &str) -> Result> { + use core::any::type_name; + match self.ty { + Some(SpecificType::Str) | None => Ok(None), + // { "foo": "Bar" } => "Bar" might be an enum variant + Some(ref mut var @ SpecificType::Variant(_, _, None)) => { + var.pick_mut(to_vec(val)?, |k| to_vec(k.name()).unwrap()) + .ok_or_else(|| Error::BadInput("Invalid variant".into()))?; + self.out.put_u8(var.variant_id()); + Ok(Some(())) + } + Some(SpecificType::StructNewType(ty)) => match self.resolve(ty) { + // { "foo": "bar" } => "bar" might be a string wrapped in a type + SpecificType::Str => Ok(None), + _ => Err(Error::NotSupported(type_name::<&str>())), + }, + Some(_) => Err(Error::NotSupported(type_name::<&str>())), + } + } } /// @@ -360,7 +435,7 @@ impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, Some(var @ Variant(_, _, Some(_))) => match (&var).into() { EnumVariant::Tuple(_, _, types) => Self::Composite(ser, types), EnumVariant::Struct(_, _, types) => { - Self::Composite(ser, types.iter().map(|(_, ty)| ty.clone()).collect()) + Self::Composite(ser, types.iter().map(|(_, ty)| *ty).collect()) } _ => Self::Empty(ser), }, @@ -401,6 +476,7 @@ where let key_data = to_vec(key)?; // assume the key is the name of the variant var.pick_mut(key_data, |v| to_vec(v.name()).unwrap()) + .ok_or_else(|| Error::BadInput("Invalid variant".into()))? .variant_id() .serialize(&mut **ser)?; } @@ -575,12 +651,22 @@ where #[derive(Debug)] pub enum Error { Ser(String), + BadInput(String), + Type(scale_info::Type), + NotSupported(&'static str), } impl fmt::Display for Error { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Error::Ser(msg) => write!(f, "{}", msg), + Error::BadInput(msg) => write!(f, "Bad Input: {}", msg), + Error::Type(ty) => write!( + f, + "Unexpected type: {}", + ty.path().ident().unwrap_or_else(|| "Unknown".into()) + ), + Error::NotSupported(from) => write!(f, "Serializing {} as type is not supported", from), } } } @@ -984,4 +1070,38 @@ mod tests { assert_eq!(out, expected); Ok(()) } + + #[test] + fn test_unordered_iter() -> Result<()> { + #[derive(Debug, Encode, TypeInfo, Serialize)] + enum Bar { + _This, + That(i16), + } + #[derive(Debug, Encode, TypeInfo, Serialize)] + struct Foo { + bar: Bar, + baz: Option, + bam: String, + } + let foo = Foo { + bar: Bar::That(i16::MAX), + baz: Some(123), + bam: "lorem ipsum".into(), + }; + let (ty, reg) = register(&foo); + + let input = vec![ + ("bam", crate::JsonValue::String("lol".into())), + ("baz", 123.into()), + ("bam", "lorem ipsum".into()), + ("bar", serde_json::json!({ "That": i16::MAX })), + ]; + + let out = to_vec_from_iter(input, (®, ty))?; + let expected = foo.encode(); + + assert_eq!(out, expected); + Ok(()) + } } From 44c746025deb33d1a10effe96fa380e41273982c Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 6 Dec 2021 09:11:11 +0100 Subject: [PATCH 44/59] Serialize hex strings to bytes (#6) --- Cargo.toml | 3 +- src/lib.rs | 13 +++++---- src/serializer.rs | 70 ++++++++++++++++++++++++++++++++++++++++------- src/value.rs | 8 ++++-- 4 files changed, 76 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 49438f8..96b220f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,9 +13,10 @@ scale-info = { version = "1.0.0", default-features = false, features = ["serde"] serde = { version = "1.0.130", default-features = false } serde_json = { version = "1.0.72", default-features = false, optional = true } codec = { version = "2.3.1", package = "parity-scale-codec", default-features = false, optional = true } +hex = { version = "0.4.3", default-features = false, features = ["alloc"], optional = true } [features] -default = ["std", "codec", "json", "experimental-serializer"] +default = ["std", "codec", "json", "hex", "experimental-serializer"] std = ["scale-info/std", "bytes/std"] experimental-serializer = [] json = ["serde_json/preserve_order"] diff --git a/src/lib.rs b/src/lib.rs index c13edaa..645d513 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -51,9 +51,9 @@ pub enum SpecificType { Bool, U8, U16, U32, U64, U128, I8, I16, I32, I64, I128, - Bytes, Char, Str, + Bytes(TypeId), Sequence(TypeId), Map(TypeId, TypeId), Tuple(TupleOrArray), @@ -116,11 +116,14 @@ impl From<(&Type, &PortableRegistry)> for SpecificType { } Def::Variant(v) => Self::Variant(name.into(), v.variants().into(), None), Def::Sequence(s) => { - let ty = resolve!(s.type_param()); - if ty.path().segments() != ["u8"] { - Self::Sequence(s.type_param().id()) + let ty = s.type_param(); + if matches!( + resolve!(ty).type_def(), + Def::Primitive(TypeDefPrimitive::U8) + ) { + Self::Bytes(ty.id()) } else { - Self::Bytes + Self::Sequence(ty.id()) } } Def::Array(a) => Self::Tuple(TupleOrArray::Array(a.type_param().id(), a.len())), diff --git a/src/serializer.rs b/src/serializer.rs index 8ba2dcd..b200cba 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -206,8 +206,8 @@ where } fn serialize_u64(self, v: u64) -> Result { - // all numbers in serde_json are the same self.maybe_some()?; + // all numbers in serde_json are the same match self.ty { Some(SpecificType::I8) => self.serialize_i8(v as i8)?, Some(SpecificType::I16) => self.serialize_i16(v as i16)?, @@ -247,6 +247,8 @@ where fn serialize_bytes(self, v: &[u8]) -> Result { self.maybe_some()?; + + compact_number(v.len(), &mut self.out); self.out.put(v); Ok(()) } @@ -309,7 +311,10 @@ where fn serialize_seq(self, len: Option) -> Result { self.maybe_some()?; - if matches!(self.ty, None | Some(SpecificType::Sequence(_))) { + if matches!( + self.ty, + None | Some(SpecificType::Bytes(_)) | Some(SpecificType::Sequence(_)) + ) { compact_number(len.expect("known length"), &mut self.out); } Ok(self.into()) @@ -392,7 +397,6 @@ where #[inline] fn maybe_other(&mut self, val: &str) -> Result> { - use core::any::type_name; match self.ty { Some(SpecificType::Str) | None => Ok(None), // { "foo": "Bar" } => "Bar" might be an enum variant @@ -405,9 +409,26 @@ where Some(SpecificType::StructNewType(ty)) => match self.resolve(ty) { // { "foo": "bar" } => "bar" might be a string wrapped in a type SpecificType::Str => Ok(None), - _ => Err(Error::NotSupported(type_name::<&str>())), + ref ty => Err(Error::NotSupported( + type_name_of_val(val), + format!("{:?}", ty), + )), }, - Some(_) => Err(Error::NotSupported(type_name::<&str>())), + #[cfg(feature = "hex")] + Some(SpecificType::Bytes(_)) => { + if val.starts_with("0x") { + let bytes = + hex::decode(&val[2..]).map_err(|e| Error::BadInput(e.to_string()))?; + ser::Serializer::serialize_bytes(self, &bytes)?; + Ok(Some(())) + } else { + Err(Error::BadInput("Hex string must start with 0x".into())) + } + } + Some(ref ty) => Err(Error::NotSupported( + type_name_of_val(val), + format!("{:?}", ty), + )), } } } @@ -430,7 +451,7 @@ impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, Some(StructTuple(fields)) => Self::Composite(ser, fields), Some(Tuple(TupleOrArray::Array(ty, _))) => Self::Sequence(ser, ty), Some(Tuple(TupleOrArray::Tuple(fields))) => Self::Composite(ser, fields), - Some(Sequence(ty)) => Self::Sequence(ser, ty), + Some(Sequence(ty) | Bytes(ty)) => Self::Sequence(ser, ty), Some(Map(_, _)) => Self::Empty(ser), Some(var @ Variant(_, _, Some(_))) => match (&var).into() { EnumVariant::Tuple(_, _, types) => Self::Composite(ser, types), @@ -653,7 +674,7 @@ pub enum Error { Ser(String), BadInput(String), Type(scale_info::Type), - NotSupported(&'static str), + NotSupported(&'static str, String), } impl fmt::Display for Error { @@ -666,7 +687,9 @@ impl fmt::Display for Error { "Unexpected type: {}", ty.path().ident().unwrap_or_else(|| "Unknown".into()) ), - Error::NotSupported(from) => write!(f, "Serializing {} as type is not supported", from), + Error::NotSupported(from, to) => { + write!(f, "Serializing {} as {} is not supported", from, to) + } } } } @@ -682,7 +705,7 @@ impl ser::Error for Error { } } -// from https://github.com/paritytech/parity-scale-codec/blob/master/src/compact.rs#L336 +// adapted from https://github.com/paritytech/parity-scale-codec/blob/master/src/compact.rs#L336 #[allow(clippy::all)] fn compact_number(n: usize, mut dest: impl BufMut) { match n { @@ -709,6 +732,11 @@ fn compact_number(n: usize, mut dest: impl BufMut) { } } +// nightly only +fn type_name_of_val(_val: &T) -> &'static str { + core::any::type_name::() +} + #[cfg(test)] mod tests { use super::*; @@ -1058,7 +1086,7 @@ mod tests { let input = Foo { bar: [Bar::That(i16::MAX), Bar::This].into(), baz: Some(Baz("lorem ipsum".into())), - lol: b"\0xFFsome stuff\0x00", + lol: b"\xFFsome stuff\x00", }; let mut out = Vec::::new(); let expected = input.encode(); @@ -1104,4 +1132,26 @@ mod tests { assert_eq!(out, expected); Ok(()) } + + #[test] + fn test_bytes_as_hex_string() -> Result<()> { + #[derive(Debug, Encode, TypeInfo, Serialize)] + struct Foo { + bar: Vec, + } + let foo = Foo { + bar: b"\x00\x12\x34\x56".to_vec(), + }; + let (ty, reg) = register(&foo); + + let hex_string = "0x00123456"; + + let input = vec![("bar", crate::JsonValue::String(hex_string.into()))]; + + let out = to_vec_from_iter(input, (®, ty))?; + let expected = foo.encode(); + + assert_eq!(out, expected); + Ok(()) + } } diff --git a/src/value.rs b/src/value.rs index 6949496..77e9fe3 100644 --- a/src/value.rs +++ b/src/value.rs @@ -111,7 +111,11 @@ impl<'a> Serialize for Value<'a> { I32 => ser.serialize_i32(data.get_i32_le()), I64 => ser.serialize_i64(data.get_i64_le()), I128 => ser.serialize_i128(data.get_i128_le()), - Bytes => ser.serialize_bytes(data.chunk()), + Bytes(_) => { + let (_, s) = sequence_len(data.chunk()); + data.advance(s); + ser.serialize_bytes(data.chunk()) + } Char => ser.serialize_char(char::from_u32(data.get_u32_le()).unwrap()), Str => { let (_, s) = sequence_len(data.chunk()); @@ -442,7 +446,7 @@ mod tests { #[test] fn serialize_u8array() -> Result<(), Error> { - let in_value: Vec = [2u8, u8::MAX].into(); + let in_value: Vec = [2u8; u8::MAX as usize].into(); let data = in_value.encode(); let (id, reg) = register(&in_value); From 55e7e9d2a8ff3270fdd35e128bf5f9f4dda94970 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Mon, 6 Dec 2021 16:48:22 +0100 Subject: [PATCH 45/59] Beta bump --- Cargo.toml | 2 +- README.md | 34 +++++++++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 96b220f..ef141d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "0.8.0" +version = "1.0.0-beta" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" diff --git a/README.md b/README.md index 83ba598..50d771c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,35 @@ # scales - SCALE Serialization -Making use of [type information](https://github.com/paritytech/scale-info) this library allows for conversion of SCALE encoded data(wrapped in a `scales::Value`) to any format that implements `Serialize` including dynamic types like `serde_json::Value` for example. The opposite conversion of arbitrary data(e.g. JSON) to SCALE binary format is also possible with the `serializer` feature. +Making use of [type information](https://github.com/paritytech/scale-info) this library allows +conversion to/from [SCALE](https://github.com/paritytech/parity-scale-codec) encoded data, +specially useful when conversion is for dynamic types like JSON. + +### From SCALE + +`scales::Value` wraps the raw SCALE binary data and the type id within type registry +giving you an object that can be serialized to any compatible format. + +```rust +let value = scales::Value::new(scale_binary_data, type_id, &type_registry); +serde_json::to_string(value)?; +``` + +### To SCALE + +Public methods from the `scales::serializer::*` module(feature `experimental-serializer`) +allow for a best effort conversion of dynamic types(e.g. `serde_json::Value`) to SCALE +binary format. The serializer tries to be smart when interpreting the input and convert it +to the desired format dictated by the provided type in the registry. + +```rust +// simple conversion +let scale_data = to_vec(some_serializable_input); // or to_bytes(&mut bytes, input); + +// with type info +let scale_data = to_vec_with_info(input, Some((®istry, type_id))); + +// from an unordered list of properties that make an object +let input = vec![("prop_b", 123), ("prop_a", 456)]; +let scale_data = to_vec_from_iter(input, (®istry, type_id)); +``` + From 54e21b1864a9ed85744c8393241735b88177bb0d Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Tue, 3 May 2022 19:44:21 +0200 Subject: [PATCH 46/59] Upgrade scale-info dependency --- Cargo.toml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ef141d1..321501b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "scale-serialization" description = "SCALE Serialization" -version = "1.0.0-beta" +version = "1.0.0-beta2" authors = ["Daniel Olano "] edition = "2018" repository = "https://github.com/virto-network/scales" @@ -9,10 +9,10 @@ license = "Apache-2.0" [dependencies] bytes = { version = "1.1.0", default-features = false } -scale-info = { version = "1.0.0", default-features = false, features = ["serde"] } -serde = { version = "1.0.130", default-features = false } -serde_json = { version = "1.0.72", default-features = false, optional = true } -codec = { version = "2.3.1", package = "parity-scale-codec", default-features = false, optional = true } +scale-info = { version = "2.1.1", default-features = false, features = ["serde"] } +serde = { version = "1.0.137", default-features = false } +serde_json = { version = "1.0.80", default-features = false, optional = true } +codec = { version = "3.1.2", package = "parity-scale-codec", default-features = false, optional = true } hex = { version = "0.4.3", default-features = false, features = ["alloc"], optional = true } [features] @@ -22,8 +22,8 @@ experimental-serializer = [] json = ["serde_json/preserve_order"] [dev-dependencies] -anyhow = "1.0.51" -codec = { version = "2.3.1", package = "parity-scale-codec", features = ["derive"] } -scale-info = { version = "1.0.0", default-features = false, features = ["derive"] } -serde = { version = "1.0.130", default-features = false, features = ["derive"] } -serde_json = { version = "1.0.72", default-features = false, features = ["alloc"] } +anyhow = "1.0.57" +codec = { version = "3.1.2", package = "parity-scale-codec", features = ["derive"] } +scale-info = { version = "2.1.1", default-features = false, features = ["derive"] } +serde = { version = "1.0.137", default-features = false, features = ["derive"] } +serde_json = { version = "1.0.80", default-features = false, features = ["alloc"] } From 7be65a1acb6c3f89b4ed7613a7572d2ca8383d40 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 10 Aug 2022 20:58:44 +0200 Subject: [PATCH 47/59] Add u128 support --- src/serializer.rs | 86 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 83 insertions(+), 3 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index b200cba..3edbf09 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -215,9 +215,23 @@ where Some(SpecificType::U8) => self.serialize_u8(v as u8)?, Some(SpecificType::U16) => self.serialize_u16(v as u16)?, Some(SpecificType::U32) => self.serialize_u32(v as u32)?, - _ => { - self.out.put_u64_le(v); - } + _ => self.out.put_u64_le(v), + } + Ok(()) + } + + fn serialize_u128(self, v: u128) -> Result { + self.maybe_some()?; + match self.ty { + Some(SpecificType::I8) => self.serialize_i8(v as i8)?, + Some(SpecificType::I16) => self.serialize_i16(v as i16)?, + Some(SpecificType::I32) => self.serialize_i32(v as i32)?, + Some(SpecificType::I64) => self.serialize_i64(v as i64)?, + Some(SpecificType::U8) => self.serialize_u8(v as u8)?, + Some(SpecificType::U16) => self.serialize_u16(v as u16)?, + Some(SpecificType::U32) => self.serialize_u32(v as u32)?, + Some(SpecificType::U64) => self.serialize_u64(v as u64)?, + _ => self.out.put_u128_le(v), } Ok(()) } @@ -414,6 +428,46 @@ where format!("{:?}", ty), )), }, + Some(SpecificType::U8) => { + let n = val.parse().map_err(|_| Error::BadInput("u8".into()))?; + Ok(Some(self.out.put_u8(n))) + } + Some(SpecificType::U16) => { + let n = val.parse().map_err(|_| Error::BadInput("u16".into()))?; + Ok(Some(self.out.put_u16_le(n))) + } + Some(SpecificType::U32) => { + let n = val.parse().map_err(|_| Error::BadInput("u32".into()))?; + Ok(Some(self.out.put_u32_le(n))) + } + Some(SpecificType::U64) => { + let n = val.parse().map_err(|_| Error::BadInput("u64".into()))?; + Ok(Some(self.out.put_u64_le(n))) + } + Some(SpecificType::U128) => { + let n = val.parse().map_err(|_| Error::BadInput("u128".into()))?; + Ok(Some(self.out.put_u128_le(n))) + } + Some(SpecificType::I8) => { + let n = val.parse().map_err(|_| Error::BadInput("i8".into()))?; + Ok(Some(self.out.put_i8(n))) + } + Some(SpecificType::I16) => { + let n = val.parse().map_err(|_| Error::BadInput("i16".into()))?; + Ok(Some(self.out.put_i16_le(n))) + } + Some(SpecificType::I32) => { + let n = val.parse().map_err(|_| Error::BadInput("i32".into()))?; + Ok(Some(self.out.put_i32_le(n))) + } + Some(SpecificType::I64) => { + let n = val.parse().map_err(|_| Error::BadInput("i64".into()))?; + Ok(Some(self.out.put_i64_le(n))) + } + Some(SpecificType::I128) => { + let n = val.parse().map_err(|_| Error::BadInput("i128".into()))?; + Ok(Some(self.out.put_i128_le(n))) + } #[cfg(feature = "hex")] Some(SpecificType::Bytes(_)) => { if val.starts_with("0x") { @@ -792,6 +846,18 @@ mod tests { Ok(()) } + #[test] + fn primitive_u128() -> Result<()> { + const INPUT: u128 = 0xFF_EE_DD_CC__BB_AA_99_88__77_66_55_44__33_22_11_00; + let mut out = [0u8; size_of::()]; + let expected = INPUT.encode(); + + to_bytes(out.as_mut(), &INPUT)?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + #[test] fn primitive_i16() -> Result<()> { const INPUT: i16 = i16::MIN; @@ -998,6 +1064,20 @@ mod tests { (sym.id(), reg.into()) } + #[test] + fn str_as_u128() -> Result<()> { + const INPUT: &str = "340282366920938463463374607431768211455"; + let mut out = [0u8; size_of::()]; + let expected = u128::MAX.encode(); + + let (id, reg) = register(&0u128); + + to_bytes_with_info(out.as_mut(), &INPUT, Some((®, id)))?; + + assert_eq!(out.as_mut(), expected); + Ok(()) + } + #[test] fn json_simple() -> Result<()> { #[derive(Debug, Serialize, Encode, TypeInfo)] From 2e1cb416c12638472a073de96272423b21d9c7c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Fri, 9 Dec 2022 20:17:35 -0500 Subject: [PATCH 48/59] feature(serializer): support encoding for number primitives under Compact --- src/lib.rs | 3 ++- src/serializer.rs | 33 ++++++++++++++++++++++++++++++++- src/value.rs | 23 +++++++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 645d513..13caa0f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -59,6 +59,7 @@ pub enum SpecificType { Tuple(TupleOrArray), Struct(Vec<(String, TypeId)>), StructUnit, StructNewType(TypeId), StructTuple(Vec), Variant(String, Vec, Option), + Compact(TypeId), } impl From<(&Type, &PortableRegistry)> for SpecificType { @@ -147,7 +148,7 @@ impl From<(&Type, &PortableRegistry)> for SpecificType { TypeDefPrimitive::U256 => unimplemented!(), TypeDefPrimitive::I256 => unimplemented!(), }, - Def::Compact(_c) => todo!(), + Def::Compact(c) => Self::Compact(c.type_param().id()), Def::BitSequence(_b) => todo!(), } } diff --git a/src/serializer.rs b/src/serializer.rs index 3edbf09..2f7c867 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,7 +1,8 @@ use crate::prelude::*; use bytes::BufMut; +use codec::Encode; use core::fmt; -use scale_info::{PortableRegistry, TypeInfo}; +use scale_info::{PortableRegistry, TypeDefPrimitive, TypeInfo}; use serde::{ser, Serialize}; use crate::{EnumVariant, SpecificType, TupleOrArray}; @@ -133,6 +134,34 @@ where }; Serializer { out, ty, registry } } + + fn serialize_compact(&mut self, ty: u32, v: u128) -> Result<()> { + let type_def = self + .registry + .expect("registry not present") + .resolve(ty) + .expect("type T for Compact not found in registry") + .type_def(); + + let compact_buffer = if let scale_info::TypeDef::Primitive(p) = type_def { + use codec::Compact; + if matches!(p, TypeDefPrimitive::U32) { + Compact(v as u32).encode() + } else if matches!(p, TypeDefPrimitive::U64) { + Compact(v as u64).encode() + } else if matches!(p, TypeDefPrimitive::U128) { + Compact(v).encode() + } else { + todo!() + } + } else { + todo!() + }; + + self.out.put_slice(&compact_buffer[..]); + + Ok(()) + } } impl<'a, 'reg, B> ser::Serializer for &'a mut Serializer<'reg, B> @@ -215,6 +244,7 @@ where Some(SpecificType::U8) => self.serialize_u8(v as u8)?, Some(SpecificType::U16) => self.serialize_u16(v as u16)?, Some(SpecificType::U32) => self.serialize_u32(v as u32)?, + Some(SpecificType::Compact(ty)) => self.serialize_compact(ty, v as u128)?, _ => self.out.put_u64_le(v), } Ok(()) @@ -231,6 +261,7 @@ where Some(SpecificType::U16) => self.serialize_u16(v as u16)?, Some(SpecificType::U32) => self.serialize_u32(v as u32)?, Some(SpecificType::U64) => self.serialize_u64(v as u64)?, + Some(SpecificType::Compact(ty)) => self.serialize_compact(ty, v as u128)?, _ => self.out.put_u128_le(v), } Ok(()) diff --git a/src/value.rs b/src/value.rs index 77e9fe3..034e90c 100644 --- a/src/value.rs +++ b/src/value.rs @@ -1,6 +1,7 @@ use crate::{EnumVariant, SpecificType}; use alloc::{collections::BTreeMap, vec::Vec}; use bytes::{Buf, Bytes}; +use codec::Encode; use core::{convert::TryInto, str}; use scale_info::{prelude::*, PortableRegistry, TypeDefPrimitive as Primitive}; use serde::ser::{SerializeMap, SerializeSeq, SerializeTuple, SerializeTupleStruct}; @@ -111,6 +112,28 @@ impl<'a> Serialize for Value<'a> { I32 => ser.serialize_i32(data.get_i32_le()), I64 => ser.serialize_i64(data.get_i64_le()), I128 => ser.serialize_i128(data.get_i128_le()), + Compact(ty) => { + let type_def = self + .registry + .resolve(ty) + .expect("not found in registry") + .type_def(); + + if let scale_info::TypeDef::Primitive(p) = type_def { + use codec::Compact; + if matches!(p, Primitive::U32) { + ser.serialize_bytes(&Compact(data.get_u32_le()).encode()) + } else if matches!(p, Primitive::U64) { + ser.serialize_bytes(&Compact(data.get_u64_le()).encode()) + } else if matches!(p, Primitive::U128) { + ser.serialize_bytes(&Compact(data.get_u128_le()).encode()) + } else { + unimplemented!() + } + } else { + unimplemented!() + } + }, Bytes(_) => { let (_, s) = sequence_len(data.chunk()); data.advance(s); From c8b2e3ebf59941d85cf0342d285f0fc42b9de70a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Tue, 13 Dec 2022 20:17:27 -0500 Subject: [PATCH 49/59] change(serializer): better assert type_def on serialize_compact --- src/serializer.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index 2f7c867..5c04046 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -143,17 +143,14 @@ where .expect("type T for Compact not found in registry") .type_def(); - let compact_buffer = if let scale_info::TypeDef::Primitive(p) = type_def { - use codec::Compact; - if matches!(p, TypeDefPrimitive::U32) { - Compact(v as u32).encode() - } else if matches!(p, TypeDefPrimitive::U64) { - Compact(v as u64).encode() - } else if matches!(p, TypeDefPrimitive::U128) { - Compact(v).encode() - } else { - todo!() - } + use scale_info::TypeDef::Primitive; + use codec::Compact; + let compact_buffer = if matches!(type_def, Primitive(TypeDefPrimitive::U32)) { + Compact(v as u32).encode() + } else if matches!(type_def, Primitive(TypeDefPrimitive::U64)) { + Compact(v as u64).encode() + } else if matches!(type_def, Primitive(TypeDefPrimitive::U128)) { + Compact(v).encode() } else { todo!() }; From 7d1c3f5d635e8df424ef6578d51cf439b459cde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Tue, 13 Dec 2022 20:36:14 -0500 Subject: [PATCH 50/59] change(serializer): on serialize_compact use self.resolve to resolve types --- src/serializer.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index 5c04046..4bdf258 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -2,7 +2,7 @@ use crate::prelude::*; use bytes::BufMut; use codec::Encode; use core::fmt; -use scale_info::{PortableRegistry, TypeDefPrimitive, TypeInfo}; +use scale_info::{PortableRegistry, TypeInfo}; use serde::{ser, Serialize}; use crate::{EnumVariant, SpecificType, TupleOrArray}; @@ -136,20 +136,14 @@ where } fn serialize_compact(&mut self, ty: u32, v: u128) -> Result<()> { - let type_def = self - .registry - .expect("registry not present") - .resolve(ty) - .expect("type T for Compact not found in registry") - .type_def(); - - use scale_info::TypeDef::Primitive; + let type_def = self.resolve(ty); + use codec::Compact; - let compact_buffer = if matches!(type_def, Primitive(TypeDefPrimitive::U32)) { + let compact_buffer = if matches!(type_def, SpecificType::U32) { Compact(v as u32).encode() - } else if matches!(type_def, Primitive(TypeDefPrimitive::U64)) { + } else if matches!(type_def, SpecificType::U64) { Compact(v as u64).encode() - } else if matches!(type_def, Primitive(TypeDefPrimitive::U128)) { + } else if matches!(type_def, SpecificType::U128) { Compact(v).encode() } else { todo!() From f3a47126ef581147b083f25059dd0b2e3660d13b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Wed, 14 Dec 2022 00:15:20 -0500 Subject: [PATCH 51/59] change(serializer): derive Debug on Serializer --- src/serializer.rs | 50 ++++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 18 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index 4bdf258..45d4dcd 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -1,7 +1,7 @@ use crate::prelude::*; use bytes::BufMut; use codec::Encode; -use core::fmt; +use core::fmt::{self, Debug}; use scale_info::{PortableRegistry, TypeInfo}; use serde::{ser, Serialize}; @@ -39,7 +39,7 @@ where pub fn to_bytes(bytes: B, value: &T) -> Result<()> where T: Serialize + ?Sized, - B: BufMut, + B: BufMut + Debug, { to_bytes_with_info(bytes, value, None) } @@ -51,7 +51,7 @@ pub fn to_bytes_with_info( ) -> Result<()> where T: Serialize + ?Sized, - B: BufMut, + B: BufMut + Debug, { let mut serializer = Serializer::new(bytes, registry_type); value.serialize(&mut serializer)?; @@ -65,7 +65,7 @@ pub fn to_bytes_from_iter( registry_type: (&PortableRegistry, TypeId), ) -> Result<()> where - B: BufMut, + B: BufMut + Debug, I: IntoIterator, K: Into, V: Into, @@ -112,7 +112,11 @@ where /// A serializer that encodes types to SCALE with the option to coerce /// the output to an equivalent representation given by some type information. -pub struct Serializer<'reg, B> { +#[derive(Debug)] +pub struct Serializer<'reg, B> +where + B: Debug +{ out: B, ty: Option, registry: Option<&'reg PortableRegistry>, @@ -120,7 +124,7 @@ pub struct Serializer<'reg, B> { impl<'reg, B> Serializer<'reg, B> where - B: BufMut, + B: BufMut + Debug, { pub fn new(out: B, registry_type: Option<(&'reg PortableRegistry, TypeId)>) -> Self { let (registry, ty) = match registry_type.map(|(reg, ty_id)| { @@ -157,7 +161,7 @@ where impl<'a, 'reg, B> ser::Serializer for &'a mut Serializer<'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -410,7 +414,7 @@ where impl<'a, 'reg, B> Serializer<'reg, B> where - B: BufMut, + B: BufMut + Debug, { // A check to run for every serialize fn since any type could be an Option::Some // if the type info says its an Option assume its Some and extract the inner type @@ -510,14 +514,20 @@ where } /// -pub enum TypedSerializer<'a, 'reg, B> { +pub enum TypedSerializer<'a, 'reg, B> +where + B: Debug +{ Empty(&'a mut Serializer<'reg, B>), Composite(&'a mut Serializer<'reg, B>, Vec), Sequence(&'a mut Serializer<'reg, B>, TypeId), Enum(&'a mut Serializer<'reg, B>), } -impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, 'reg, B> { +impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, 'reg, B> +where + B: Debug +{ fn from(ser: &'a mut Serializer<'reg, B>) -> Self { use SpecificType::*; match ser.ty.take() { @@ -545,7 +555,11 @@ impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, } } -impl<'a, 'reg, B> TypedSerializer<'a, 'reg, B> { + +impl<'a, 'reg, B> TypedSerializer<'a, 'reg, B> +where + B: Debug +{ fn serializer(&mut self) -> &mut Serializer<'reg, B> { match self { Self::Empty(ser) @@ -558,7 +572,7 @@ impl<'a, 'reg, B> TypedSerializer<'a, 'reg, B> { impl<'a, 'reg, B> ser::SerializeMap for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -616,7 +630,7 @@ where impl<'a, 'reg, B> ser::SerializeSeq for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -652,7 +666,7 @@ where impl<'a, 'reg, B> ser::SerializeStruct for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -671,7 +685,7 @@ where impl<'a, 'reg, B> ser::SerializeStructVariant for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -690,7 +704,7 @@ where impl<'a, 'reg, B> ser::SerializeTuple for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -709,7 +723,7 @@ where impl<'a, 'reg, B> ser::SerializeTupleStruct for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; @@ -728,7 +742,7 @@ where impl<'a, 'reg, B> ser::SerializeTupleVariant for TypedSerializer<'a, 'reg, B> where - B: BufMut, + B: BufMut + Debug, { type Ok = (); type Error = Error; From 9ff8b3d0ff02af3631fdc68c1514d88116adb546 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Fri, 16 Dec 2022 23:16:52 +0100 Subject: [PATCH 52/59] Replace if matches! with match --- src/serializer.rs | 24 ++++++++++-------------- src/value.rs | 27 +++++++++++++-------------- 2 files changed, 23 insertions(+), 28 deletions(-) diff --git a/src/serializer.rs b/src/serializer.rs index 45d4dcd..0cfa496 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -115,7 +115,7 @@ where #[derive(Debug)] pub struct Serializer<'reg, B> where - B: Debug + B: Debug, { out: B, ty: Option, @@ -143,18 +143,15 @@ where let type_def = self.resolve(ty); use codec::Compact; - let compact_buffer = if matches!(type_def, SpecificType::U32) { - Compact(v as u32).encode() - } else if matches!(type_def, SpecificType::U64) { - Compact(v as u64).encode() - } else if matches!(type_def, SpecificType::U128) { - Compact(v).encode() - } else { - todo!() + let compact_buffer = match type_def { + SpecificType::U32 => Compact(v as u32).encode(), + SpecificType::U64 => Compact(v as u64).encode(), + SpecificType::U128 => Compact(v).encode(), + _ => todo!(), }; self.out.put_slice(&compact_buffer[..]); - + Ok(()) } } @@ -516,7 +513,7 @@ where /// pub enum TypedSerializer<'a, 'reg, B> where - B: Debug + B: Debug, { Empty(&'a mut Serializer<'reg, B>), Composite(&'a mut Serializer<'reg, B>, Vec), @@ -526,7 +523,7 @@ where impl<'a, 'reg, B: 'a> From<&'a mut Serializer<'reg, B>> for TypedSerializer<'a, 'reg, B> where - B: Debug + B: Debug, { fn from(ser: &'a mut Serializer<'reg, B>) -> Self { use SpecificType::*; @@ -555,10 +552,9 @@ where } } - impl<'a, 'reg, B> TypedSerializer<'a, 'reg, B> where - B: Debug + B: Debug, { fn serializer(&mut self) -> &mut Serializer<'reg, B> { match self { diff --git a/src/value.rs b/src/value.rs index 034e90c..585209f 100644 --- a/src/value.rs +++ b/src/value.rs @@ -119,21 +119,20 @@ impl<'a> Serialize for Value<'a> { .expect("not found in registry") .type_def(); - if let scale_info::TypeDef::Primitive(p) = type_def { - use codec::Compact; - if matches!(p, Primitive::U32) { - ser.serialize_bytes(&Compact(data.get_u32_le()).encode()) - } else if matches!(p, Primitive::U64) { - ser.serialize_bytes(&Compact(data.get_u64_le()).encode()) - } else if matches!(p, Primitive::U128) { - ser.serialize_bytes(&Compact(data.get_u128_le()).encode()) - } else { - unimplemented!() - } - } else { - unimplemented!() + use codec::Compact; + match type_def { + TypeDef::Primitive(Primitive::U32) => { + ser.serialize_bytes(&Compact(data.get_u32_le()).encode()) } - }, + TypeDef::Primitive(Primitive::U64) => { + ser.serialize_bytes(&Compact(data.get_u64_le()).encode()) + } + TypeDef::Primitive(Primitive::U128) => { + ser.serialize_bytes(&Compact(data.get_u128_le()).encode()) + } + _ => unimplemented!(), + } + } Bytes(_) => { let (_, s) = sequence_len(data.chunk()); data.advance(s); From 62f05070b1592e44b24c36133870f4b66be293fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Tue, 20 Dec 2022 15:19:21 -0500 Subject: [PATCH 53/59] change(serializer): set TypedSerializer as debuggable --- src/serializer.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/serializer.rs b/src/serializer.rs index 0cfa496..f0c87a9 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -511,6 +511,7 @@ where } /// +#[derive(Debug)] pub enum TypedSerializer<'a, 'reg, B> where B: Debug, From 5f0f6e4e2b9bf71953075bbce11cb81660ae1500 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Tue, 20 Dec 2022 16:28:33 -0500 Subject: [PATCH 54/59] fix(serializer): on serialize_value, unwrap StructNewType(ty_id) for Enum-related values --- src/serializer.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/serializer.rs b/src/serializer.rs index f0c87a9..25a8b59 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -611,7 +611,14 @@ where TypedSerializer::Enum(ser) => { if let Some(var @ SpecificType::Variant(_, _, Some(_))) = &ser.ty { if let EnumVariant::NewType(_, _, ty_id) = var.into() { - ser.ty = Some(ser.resolve(ty_id)); + let ty = ser.resolve(ty_id); + + ser.ty = Some(if let SpecificType::StructNewType(ty_id) = ty { + let ty = ser.resolve(ty_id); + ty + } else { + ty + }); } } } From cc3ae2a07bbcb9938c8700a9197d37430c564931 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 21 Dec 2022 09:22:31 +0100 Subject: [PATCH 55/59] Make clippy happy --- src/lib.rs | 14 ++++++------- src/serializer.rs | 50 +++++++++++++++++++++++++++-------------------- src/value.rs | 12 ++++++------ 3 files changed, 42 insertions(+), 34 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 13caa0f..73d9dc6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -115,7 +115,7 @@ impl From<(&Type, &PortableRegistry)> for SpecificType { ) } } - Def::Variant(v) => Self::Variant(name.into(), v.variants().into(), None), + Def::Variant(v) => Self::Variant(name, v.variants().into(), None), Def::Sequence(s) => { let ty = s.type_param(); if matches!( @@ -181,7 +181,7 @@ impl SpecificType { SpecificType::Variant(_, ref mut variants, idx @ None) => { let i = variants .iter() - .map(|v| get_field(v)) + .map(get_field) .position(|f| f.as_ref() == selection.as_ref())? as u8; variants.retain(|v| v.index() == i); *idx = Some(i); @@ -222,26 +222,26 @@ impl<'a> From<&'a SpecificType> for EnumVariant<'a> { if name == "Option" && vname == "None" { Self::OptionNone } else { - Self::Unit(*idx, &vname) + Self::Unit(*idx, vname) } } else if is_tuple!(variant) { if fields.len() == 1 { let ty = fields.first().map(|f| f.ty().id()).unwrap(); - return if name == "Option" && variant.name() == &"Some" { + return if name == "Option" && variant.name() == "Some" { Self::OptionSome(ty) } else { - Self::NewType(*idx, &vname, ty) + Self::NewType(*idx, vname, ty) }; } else { let fields = fields.iter().map(|f| f.ty().id()).collect(); - Self::Tuple(*idx, &vname, fields) + Self::Tuple(*idx, vname, fields) } } else { let fields = fields .iter() .map(|f| (f.name().unwrap().deref(), f.ty().id())) .collect(); - Self::Struct(*idx, &vname, fields) + Self::Struct(*idx, vname, fields) } } _ => panic!("Only for enum variants"), diff --git a/src/serializer.rs b/src/serializer.rs index 25a8b59..e805952 100644 --- a/src/serializer.rs +++ b/src/serializer.rs @@ -253,7 +253,7 @@ where Some(SpecificType::U16) => self.serialize_u16(v as u16)?, Some(SpecificType::U32) => self.serialize_u32(v as u32)?, Some(SpecificType::U64) => self.serialize_u64(v as u64)?, - Some(SpecificType::Compact(ty)) => self.serialize_compact(ty, v as u128)?, + Some(SpecificType::Compact(ty)) => self.serialize_compact(ty, v)?, _ => self.out.put_u128_le(v), } Ok(()) @@ -409,7 +409,7 @@ where } } -impl<'a, 'reg, B> Serializer<'reg, B> +impl<'reg, B> Serializer<'reg, B> where B: BufMut + Debug, { @@ -453,49 +453,58 @@ where }, Some(SpecificType::U8) => { let n = val.parse().map_err(|_| Error::BadInput("u8".into()))?; - Ok(Some(self.out.put_u8(n))) + self.out.put_u8(n); + Ok(Some(())) } Some(SpecificType::U16) => { let n = val.parse().map_err(|_| Error::BadInput("u16".into()))?; - Ok(Some(self.out.put_u16_le(n))) + self.out.put_u16_le(n); + Ok(Some(())) } Some(SpecificType::U32) => { let n = val.parse().map_err(|_| Error::BadInput("u32".into()))?; - Ok(Some(self.out.put_u32_le(n))) + self.out.put_u32_le(n); + Ok(Some(())) } Some(SpecificType::U64) => { let n = val.parse().map_err(|_| Error::BadInput("u64".into()))?; - Ok(Some(self.out.put_u64_le(n))) + self.out.put_u64_le(n); + Ok(Some(())) } Some(SpecificType::U128) => { let n = val.parse().map_err(|_| Error::BadInput("u128".into()))?; - Ok(Some(self.out.put_u128_le(n))) + self.out.put_u128_le(n); + Ok(Some(())) } Some(SpecificType::I8) => { let n = val.parse().map_err(|_| Error::BadInput("i8".into()))?; - Ok(Some(self.out.put_i8(n))) + self.out.put_i8(n); + Ok(Some(())) } Some(SpecificType::I16) => { let n = val.parse().map_err(|_| Error::BadInput("i16".into()))?; - Ok(Some(self.out.put_i16_le(n))) + self.out.put_i16_le(n); + Ok(Some(())) } Some(SpecificType::I32) => { let n = val.parse().map_err(|_| Error::BadInput("i32".into()))?; - Ok(Some(self.out.put_i32_le(n))) + self.out.put_i32_le(n); + Ok(Some(())) } Some(SpecificType::I64) => { let n = val.parse().map_err(|_| Error::BadInput("i64".into()))?; - Ok(Some(self.out.put_i64_le(n))) + self.out.put_i64_le(n); + Ok(Some(())) } Some(SpecificType::I128) => { let n = val.parse().map_err(|_| Error::BadInput("i128".into()))?; - Ok(Some(self.out.put_i128_le(n))) + self.out.put_i128_le(n); + Ok(Some(())) } #[cfg(feature = "hex")] Some(SpecificType::Bytes(_)) => { - if val.starts_with("0x") { - let bytes = - hex::decode(&val[2..]).map_err(|e| Error::BadInput(e.to_string()))?; + if let Some(bytes) = val.strip_prefix("0x") { + let bytes = hex::decode(bytes).map_err(|e| Error::BadInput(e.to_string()))?; ser::Serializer::serialize_bytes(self, &bytes)?; Ok(Some(())) } else { @@ -614,8 +623,7 @@ where let ty = ser.resolve(ty_id); ser.ty = Some(if let SpecificType::StructNewType(ty_id) = ty { - let ty = ser.resolve(ty_id); - ty + ser.resolve(ty_id) } else { ty }); @@ -1148,13 +1156,13 @@ mod tests { #[test] fn json_mix() -> Result<()> { #[derive(Debug, Serialize, Encode, TypeInfo)] - struct Foo<'a> { + struct Foo { a: Vec, - b: (Bar<'a>, Bar<'a>, Bar<'a>), + b: (Bar, Bar, Bar), } #[derive(Debug, Serialize, Encode, TypeInfo)] - enum Bar<'a> { - A { thing: &'a str }, + enum Bar { + A { thing: &'static str }, B(Baz), C(BTreeMap, i64), } diff --git a/src/value.rs b/src/value.rs index 585209f..b454f14 100644 --- a/src/value.rs +++ b/src/value.rs @@ -303,9 +303,9 @@ impl<'reg> core::fmt::Display for Value<'reg> { } #[cfg(feature = "json")] -impl<'reg> Into for Value<'reg> { - fn into(self) -> crate::JsonValue { - serde_json::value::to_value(self).unwrap() +impl<'reg> From> for crate::JsonValue { + fn from(val: Value<'reg>) -> Self { + serde_json::value::to_value(val).unwrap() } } @@ -628,15 +628,15 @@ mod tests { #[test] fn serialize_tuple_struct() -> Result<(), Error> { #[derive(Encode, Serialize, TypeInfo)] - struct Foo<'a>([u8; 4], (bool, Option<()>), Baz<'a>, Baz<'a>); + struct Foo([u8; 4], (bool, Option<()>), Baz, Baz); #[derive(Encode, Serialize, TypeInfo)] struct Bar; #[derive(Encode, Serialize, TypeInfo)] - enum Baz<'a> { + enum Baz { A(Bar), - B { bb: &'a str }, + B { bb: &'static str }, } let in_value = Foo( From 035d71b355c4d3b34a5884b81b3f330a82f1cd33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20Andr=C3=A9s=20Dorado=20Su=C3=A1rez?= Date: Thu, 29 Dec 2022 02:09:19 -0500 Subject: [PATCH 56/59] fix(src/value): when getting ty_len of tuples, was getting tuple lenght instead of sum of lengths of fields within tuple --- src/value.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/value.rs b/src/value.rs index b454f14..083d6cc 100644 --- a/src/value.rs +++ b/src/value.rs @@ -84,7 +84,10 @@ impl<'a> Value<'a> { (0..len).fold(prefix_size, |c, _| c + self.ty_len(&data[c..], ty_id)) } TypeDef::Array(a) => a.len().try_into().unwrap(), - TypeDef::Tuple(t) => t.fields().len(), + TypeDef::Tuple(t) => t + .fields() + .iter() + .fold(0, |c, f| c + self.ty_len(&data[c..], f.id())), TypeDef::Compact(_) => compact_len(data), TypeDef::BitSequence(_) => unimplemented!(), } From d7805670f984eae14b8ab30ac8ba783304d52295 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 17 Jan 2024 15:23:16 +0100 Subject: [PATCH 57/59] Use Rust stable --- rust-toolchain | 1 - 1 file changed, 1 deletion(-) delete mode 100644 rust-toolchain diff --git a/rust-toolchain b/rust-toolchain deleted file mode 100644 index 07ade69..0000000 --- a/rust-toolchain +++ /dev/null @@ -1 +0,0 @@ -nightly \ No newline at end of file From 0e339c86c11f428fe6bddcd0a4edf310dc0d3db6 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Wed, 17 Jan 2024 18:06:19 +0100 Subject: [PATCH 58/59] Include simple vault by default and check simplest build on no-std --- libwallet/Cargo.toml | 10 +++++----- libwallet/js/Cargo.toml | 4 ++-- libwallet/justfile | 6 ++++++ libwallet/src/key_pair.rs | 18 ++++++++++++++++-- libwallet/src/lib.rs | 19 ++----------------- libwallet/src/vault.rs | 24 +++++++++++++++++++----- libwallet/src/vault/simple.rs | 16 ++++++++-------- 7 files changed, 58 insertions(+), 39 deletions(-) create mode 100644 libwallet/justfile diff --git a/libwallet/Cargo.toml b/libwallet/Cargo.toml index 439bb25..4f1e9ec 100644 --- a/libwallet/Cargo.toml +++ b/libwallet/Cargo.toml @@ -20,7 +20,7 @@ mnemonic = {package = "bip0039", version = "0.10.1", default-features = false, o rand_core = {version = "0.6.3", optional = true} # substrate related -schnorrkel = {version = "0.10.2", default-features = false, optional = true}# soft derivation in no_std +schnorrkel = {version = "0.11.4", default-features = false, optional = true}# soft derivation in no_std rand_chacha = {version = "0.3.1", default-features = false, optional = true} # vault os @@ -35,9 +35,10 @@ serde_json = {version = "1.0", default-features = false, features = ["alloc"]} dirs = "4.0" [features] -default = ["std", "substrate", "vault_simple", "mnemonic", "rand"] -rand = ["rand_core"] -sr25519 = ["schnorrkel/u64_backend", "schnorrkel/getrandom"] +# The library has no default features but this can be uncommented during development +# default = [ "std", "substrate", "vault_simple", "vault_os", "vault_pass", "mnemonic", "util_pin", "rand", ] +rand = ["rand_core", "schnorrkel?/getrandom"] +sr25519 = ["dep:schnorrkel"] std = [ "rand_core/getrandom", ] @@ -45,7 +46,6 @@ substrate = ["sr25519"] util_pin = ["pbkdf2", "hmac", "sha2"] vault_os = ["keyring"] vault_pass = ["prs-lib"] -vault_simple = [] [workspace] members = [ diff --git a/libwallet/js/Cargo.toml b/libwallet/js/Cargo.toml index 2b4fdae..bd65dd3 100644 --- a/libwallet/js/Cargo.toml +++ b/libwallet/js/Cargo.toml @@ -25,10 +25,10 @@ wasm-bindgen-test = "0.3.34" features = ["js"] [features] -default = ["wallet", "js", "hex", "util_pin", "vault_simple"] +default = ["wallet", "js", "hex", "util_pin"] hex = ["dep:hex"] js = ["std"] std = [] util_pin = ["libwallet/util_pin"] -vault_simple = ["libwallet/vault_simple", "libwallet/mnemonic", "libwallet/rand"] +vault_simple = ["libwallet/mnemonic", "libwallet/rand"] wallet = ["libwallet/serde", "libwallet/sr25519", "libwallet/substrate"] diff --git a/libwallet/justfile b/libwallet/justfile new file mode 100644 index 0000000..f084758 --- /dev/null +++ b/libwallet/justfile @@ -0,0 +1,6 @@ +default: + just --choose + +check-no-std: + cargo build --features substrate --target wasm32-unknown-unknown + cargo build --features substrate --target riscv32i-unknown-none-elf diff --git a/libwallet/src/key_pair.rs b/libwallet/src/key_pair.rs index 1d33745..aeaf0c2 100644 --- a/libwallet/src/key_pair.rs +++ b/libwallet/src/key_pair.rs @@ -57,7 +57,10 @@ pub mod any { fn public(&self) -> Self::Public { match self { + #[cfg(feature = "sr25519")] Pair::Sr25519(p) => AnyPublic::Sr25519(p.public()), + #[cfg(not(feature = "sr25519"))] + Pair::_None => unreachable!(), } } } @@ -70,7 +73,10 @@ pub mod any { Self: Sized, { match self { + #[cfg(feature = "sr25519")] Pair::Sr25519(kp) => Pair::Sr25519(kp.derive(path)), + #[cfg(not(feature = "sr25519"))] + Pair::_None => unreachable!(), } } } @@ -94,7 +100,10 @@ pub mod any { fn verify>(&self, msg: M, sig: &[u8]) -> bool { match self { + #[cfg(feature = "sr25519")] Pair::Sr25519(p) => super::Signer::verify(p, msg, sig), + #[cfg(not(feature = "sr25519"))] + Pair::_None => unreachable!(), } } } @@ -110,7 +119,10 @@ pub mod any { impl AsRef<[u8]> for AnyPublic { fn as_ref(&self) -> &[u8] { match self { + #[cfg(feature = "sr25519")] AnyPublic::Sr25519(p) => p.as_ref(), + #[cfg(not(feature = "sr25519"))] + AnyPublic::_None => unreachable!(), } } } @@ -225,13 +237,13 @@ pub mod sr25519 { #[cfg(not(feature = "rand_chacha"))] fn derive_simple(key: SecretKey, j: Junction) -> SecretKey { - key.derived_key_simple(ChainCode(j), &[]).0 + key.derived_key_simple(ChainCode(j), []).0 } #[cfg(feature = "rand_chacha")] fn derive_simple(key: SecretKey, j: Junction) -> SecretKey { use rand_core::SeedableRng; // As noted in https://docs.rs/schnorrkel/latest/schnorrkel/context/fn.attach_rng.html - // it's not recommended by should be ok for our simple use cases + // it's not recommended but should be ok for our simple use cases let rng = rand_chacha::ChaChaRng::from_seed([0; 32]); key.derived_key_simple_rng(ChainCode(j), &[], rng).0 } @@ -356,6 +368,8 @@ mod derive { #[cfg(test)] mod tests { use super::*; + extern crate alloc; + use alloc::vec::Vec; #[test] fn substrate_junctions() { diff --git a/libwallet/src/lib.rs b/libwallet/src/lib.rs index 3aa2ec4..2f89e4c 100644 --- a/libwallet/src/lib.rs +++ b/libwallet/src/lib.rs @@ -1,6 +1,4 @@ -#![feature(async_fn_in_trait, impl_trait_projections)] -// #![feature(result_option_inspect)] -#![cfg_attr(not(feature = "std"), no_std)] +#![cfg_attr(not(any(test, feature = "std")), no_std)] //! `libwallet` is the one-stop tool to build easy, slightly opinionated crypto wallets //! that run in all kinds of environments and plattforms including embedded hardware, //! mobile apps or the Web. @@ -96,7 +94,7 @@ where def.unlock(root); }) .await - .map_err(|e| Error::Vault(e))?; + .map_err(Error::Vault)?; self.is_locked = false; } Ok(()) @@ -305,19 +303,6 @@ mod util { phrase } - macro_rules! seed_from_entropy { - ($seed: ident, $pin: expr) => { - #[cfg(feature = "util_pin")] - let protected_seed = $pin.protect::<64>($seed); - #[cfg(feature = "util_pin")] - let $seed = &protected_seed; - #[cfg(not(feature = "util_pin"))] - let _ = &$pin; // use the variable to avoid warning - }; - } - - pub(crate) use seed_from_entropy; - /// A simple pin credential that can be used to add some /// extra level of protection to seeds stored in vaults #[derive(Default, Copy, Clone)] diff --git a/libwallet/src/vault.rs b/libwallet/src/vault.rs index be95885..97f9f92 100644 --- a/libwallet/src/vault.rs +++ b/libwallet/src/vault.rs @@ -3,17 +3,15 @@ mod os; #[cfg(feature = "vault_pass")] mod pass; -#[cfg(feature = "vault_simple")] mod simple; #[cfg(feature = "vault_os")] pub use os::*; #[cfg(feature = "vault_pass")] pub use pass::*; -#[cfg(feature = "vault_simple")] pub use simple::*; -use crate::{any, key_pair, Derive}; +use crate::{any, Derive}; /// Abstration for storage of private keys that are protected by some credentials. pub trait Vault { @@ -36,14 +34,16 @@ pub trait Vault { #[derive(Debug)] pub struct RootAccount { #[cfg(feature = "substrate")] - sub: key_pair::sr25519::Pair, + sub: crate::key_pair::sr25519::Pair, } impl RootAccount { fn from_bytes(seed: &[u8]) -> Self { + #[cfg(not(feature = "substrate"))] + let _ = seed; RootAccount { #[cfg(feature = "substrate")] - sub: ::from_bytes(seed), + sub: ::from_bytes(seed), } } } @@ -61,6 +61,20 @@ impl<'a> Derive for &'a RootAccount { "m/" => unimplemented!(), #[cfg(feature = "substrate")] _ => self.sub.derive("//default").into(), + #[cfg(not(feature = "substrate"))] + _ => unreachable!(), } } } + +macro_rules! seed_from_entropy { + ($seed: ident, $pin: expr) => { + #[cfg(feature = "util_pin")] + let protected_seed = $pin.protect::<64>($seed); + #[cfg(feature = "util_pin")] + let $seed = &protected_seed; + #[cfg(not(feature = "util_pin"))] + let _ = &$pin; // use the variable to avoid warning + }; +} +pub(crate) use seed_from_entropy; diff --git a/libwallet/src/vault/simple.rs b/libwallet/src/vault/simple.rs index 1d5b022..029d94c 100644 --- a/libwallet/src/vault/simple.rs +++ b/libwallet/src/vault/simple.rs @@ -1,6 +1,5 @@ -use core::convert::TryInto; - -use crate::util::{seed_from_entropy, Pin}; +use super::seed_from_entropy; +use crate::util::Pin; use crate::{RootAccount, Vault}; /// A vault that holds secrets in memory @@ -39,20 +38,21 @@ impl Simple { R: rand_core::CryptoRng + rand_core::RngCore, { let phrase = crate::util::gen_phrase(rng, Default::default()); - ( - Self::from_phrase(&phrase), - phrase - ) + (Self::from_phrase(&phrase), phrase) } #[cfg(feature = "mnemonic")] // Provide your own seed pub fn from_phrase(phrase: impl AsRef) -> Self { + use core::convert::TryInto; let phrase = phrase .as_ref() .parse::() .expect("mnemonic"); - let entropy = phrase.entropy().try_into().expect("Size should be 32 bytes"); + let entropy = phrase + .entropy() + .try_into() + .expect("Size should be 32 bytes"); Simple { locked: Some(entropy), From 07d0030bcf075c2a1b8ff090732e0ac5a3a733e2 Mon Sep 17 00:00:00 2001 From: Daniel Olano Date: Thu, 25 Jan 2024 11:07:55 +0100 Subject: [PATCH 59/59] Create README with WIP message --- README.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..53aaeaa --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# VirtoSDK + +> ⚠️ Work in progress. The Sube repository is now the VirtoSDK where we are gathering all our client side tools and will add higher lever abstractions for applications to easily integrate Web3 services.