Fix/complex structures in url (#65)

* fix: compact value serialization

* reverse: legacy url to create backend

* fix: hash touple argument

* chore: cargo clipply

* chore: fix depednency

* chore: add from_bytes

* chore: add no_std flag to cargo

scales/src/value.rs

@@ -259,11 +259,13 @@ fn sequence_size(data: &[u8]) -> (usize, usize) {
     (
         match len {
             1 => (data[0] >> 2).into(),
-            2 => u16::from_le_bytes([(data[0] >> 2), data[1]]).into(),
-            4 => u32::from_le_bytes([(data[0] >> 2), data[1], data[2], data[3]])
+            2 => u16::from(data[0] >> 2 | data[1] << 6).into(),
+            4 => ((data[0] as u32) >> 2
+                | (data[1] as u32) << 6
+                | (data[2] as u32) << 14
+                | (data[3] as u32) << 22)
                 .try_into()
                 .unwrap(),
-
             _ => todo!(),
         },
         len,
@@ -330,6 +332,18 @@ mod tests {
     };
     use serde_json::to_value;
 
+    #[test]
+    fn test_compact_two_bytes() {
+        let data: [u8; 2] = [0x99, 0x01];
+        assert_eq!(sequence_size(&data), (102, 2));
+
+        let data: [u8; 2] = [0x15, 0x01];
+        assert_eq!(sequence_size(&data), (69, 2));
+
+        let data: [u8; 4] = [0xfe, 0xff, 0x03, 0x00];
+        assert_eq!(sequence_size(&data), (65535, 4));
+    }
+
     fn register<T>(_ty: &T) -> (u32, PortableRegistry)
     where
         T: TypeInfo + 'static,

sube/Cargo.toml

@@ -53,9 +53,10 @@ rand_core = "0.6.3"
 default = ["v14"]
 test = ["std", "wss", "http", "json", "v14", "dep:async-std", "dep:rand_core"]
 http = ["dep:jsonrpc", "surf/h1-client-rustls"]
-http-web = ["dep:jsonrpc", "dep:wasm-bindgen", "surf?/wasm-client"]
+http-web = ["dep:jsonrpc", "dep:wasm-bindgen", "surf/wasm-client"]
 json = ["scales/json"]
 std = []
+no_std = []
 
 v14 = ["dep:scale-info", "frame-metadata/current"]
 ws = ["dep:async-mutex", "dep:async-std", "dep:ewebsock", "dep:futures-channel", "dep:futures-util", "dep:jsonrpc", "async-std/unstable"]

sube/examples/query_membership.rs

@@ -0,0 +1,26 @@
+use core::future::{Future, IntoFuture};
+use serde::{Deserialize, Serialize};
+use serde_json::{from_value, Value};
+use sube::{sube, Response};
+
+#[async_std::main]
+async fn main() -> sube::Result<()> {
+    env_logger::init();
+
+    let query = format!(
+        "ws://127.0.0.1:12281/communityMemberships/account/{}/{}",
+        "0x12840f0626ac847d41089c4e05cf0719c5698af1e3bb87b66542de70b2de4b2b",
+        1
+    );
+
+    let r = sube!(&query).await?;
+
+    if let Response::ValueSet(ref v) = r {
+        let json_value = serde_json::to_value(v).expect("to be serializable");
+        println!("json: {:?}", json_value);
+        let x = serde_json::to_string_pretty(&json_value).expect("it must return an str");
+        println!("Account info: {:?}", x);
+    }
+
+    Ok(())
+}

sube/examples/query_preimage.rs

@@ -0,0 +1,26 @@
+use core::future::{Future, IntoFuture};
+use serde::{Deserialize, Serialize};
+use serde_json::{from_value, Value};
+use sube::{sube, Response};
+
+#[async_std::main]
+async fn main() -> sube::Result<()> {
+    env_logger::init();
+
+    let query = format!(
+        "ws://127.0.0.1:12281/preimage/preimageFor/{}/{}",
+        "0x6b172c3695dca229e71c0bca790f5991b68f8eee96334e842312a0a7d4a46c6c",
+        30
+    );
+
+    let r = sube!(&query).await?;
+
+    if let Response::Value(ref v) = r {
+        let json_value = serde_json::to_value(v).expect("to be serializable");
+        println!("json: {:?}", json_value);
+        let x = serde_json::to_string_pretty(&json_value).expect("it must return an str");
+        println!("Account info: {:?}", x);
+    }
+
+    Ok(())
+}

sube/examples/query_referendum_info.rs

@@ -0,0 +1,22 @@
+use env_logger;
+use serde_json;
+use sube::{sube, ExtrinsicBody, Response, Result, SubeBuilder};
+
+#[async_std::main]
+async fn main() -> Result<()> {
+    env_logger::init();
+
+    let query = format!(
+        "https://kreivo.io/communityReferenda/referendumInfoFor/{}",
+        24
+    );
+
+    let r = sube!(&query).await?;
+
+    if let Response::Value(ref v) = r {
+        let json_value = serde_json::to_value(v).expect("it must to be an valid Value");
+        println!("Raw JSON value: {:?}", json_value);
+        println!("Info: {}", serde_json::to_string_pretty(&json_value).expect("it must return an str"));
+    }
+    Ok(())
+}

sube/src/builder.rs

@@ -65,6 +65,7 @@ impl<'a> SubeBuilder<'a, (), ()> {
         let path = url.path();
 
         log::info!("building the backend for {}", url);
+
         let backend = BACKEND
             .get_or_try_init(get_backend_by_url(url.clone()))
             .await?;

sube/src/lib.rs

@@ -52,7 +52,8 @@ pub mod http;
 #[cfg(feature = "ws")]
 pub mod ws;
 
-mod builder;
+pub mod builder;
+pub use builder::SubeBuilder;
 mod hasher;
 mod meta_ext;
 mod signer;
@@ -71,11 +72,11 @@ pub fn sube(url: &str) -> builder::SubeBuilder<(), ()> {
 pub type Result<T> = core::result::Result<T, Error>;
 async fn query<'m>(chain: &impl Backend, meta: &'m Metadata, path: &str) -> Result<Response<'m>> {
     let (pallet, item_or_call, mut keys) = parse_uri(path).ok_or(Error::BadInput)?;
-
+    log::info!("pallet {}", pallet);
     let pallet = meta
         .pallet_by_name(&pallet)
         .ok_or_else(|| Error::PalletNotFound(pallet))?;
-
+    
     if item_or_call == "_constants" {
         let const_name = keys.pop().ok_or_else(|| Error::MissingConstantName)?;
         let const_meta = pallet
@@ -94,6 +95,7 @@ async fn query<'m>(chain: &impl Backend, meta: &'m Metadata, path: &str) -> Resu
     if let Ok(key_res) = StorageKey::build_with_registry(&meta.types, pallet, &item_or_call, &keys)
     {
         if !key_res.is_partial() {
+            log::info!("is not partial");
             let res = chain.query_storage(&key_res).await?;
             return Ok(Response::Value(Value::new(res, key_res.ty, &meta.types)));
         }

sube/src/meta_ext.rs

@@ -22,8 +22,8 @@ mod v14 {
     pub type Type = scale_info::Type<PortableForm>;
 }
 
-/// Decode metadata from its raw prefixed format to the currently
-/// active metadata version.
+// Decode metadata from its raw prefixed format to the currently
+// active metadata version.
 pub fn from_bytes(bytes: &mut &[u8]) -> core::result::Result<Metadata, codec::Error> {
     let meta: RuntimeMetadataPrefixed = Decode::decode(bytes)?;
     let meta = match meta.1 {
@@ -128,7 +128,7 @@ impl StorageKey {
             .as_ref()
             .and_then(|s| s.entries.iter().find(|e| e.name == item))
             .ok_or(crate::Error::StorageKeyNotFound)?;
-
+        log::trace!("map_keys={}", map_keys.iter().map(|x| x.as_ref()).collect::<Vec<&str>>().join(", "));
         entry.ty.key(registry, &meta.name, &entry.name, map_keys)
     }
 }
@@ -175,45 +175,84 @@ pub trait EntryTy {
             let key_type_info = portable_reg
                 .resolve(key_ty_id)
                 .ok_or(crate::Error::BadInput)?;
+
+            log::trace!("key_type_info={:?}", key_type_info);
             extract_touple_type(key_ty_id, key_type_info)
         } else {
             vec![]
         };
+
+        if type_call_ids.len() == hashers.len() {
+            log::trace!("type_call_ids={:?}", type_call_ids);
+            let storage_key = StorageKey::new(
+                value_ty_id,
+                hash(&Hasher::Twox128, pallet_item.0),
+                hash(&Hasher::Twox128, pallet_item.1),
+                type_call_ids
+                    .into_iter()
+                    .enumerate()
+                    .map(|(i, type_id)| {
+                        log::info!("type_call_ids.i={} type_call_ids.type_id={}", i, type_id);
+                        let k = map_keys.get(i);
+                        let hasher = hashers.get(i).expect("hasher not found");
+
+                        if k.is_none() {
+                            return KeyValue::Empty(type_id);
+                        }
+
+                        let k = k.expect("it must exist").as_ref();
+
+                        let hasher = hasher.borrow();
+                        let mut out = vec![];
+
+                        if let Some(k) = k.strip_prefix("0x") {
+                            let value = hex::decode(k).expect("str must be encoded");
+                            let _ = to_bytes_with_info(&mut out, &value, Some((portable_reg, type_id)));
+                        } else {
+                            let _ = to_bytes_with_info(&mut out, &k, Some((portable_reg, type_id)));
+                        }
+
+                        let hash = hash(hasher, &out);
+                        KeyValue::Value((type_id, hash, out, hasher.clone()))
+                    })
+                    .collect(),
+            );
+
+            Ok(storage_key)
+        } else if hashers.len() == 1 {
+            log::trace!("treating touple as argument for hasher");
+
+            let touple_hex: Vec<u8> = type_call_ids
+                    .into_iter()
+                    .enumerate()
+                    .flat_map(|(i, type_id)| {
+                        let k = map_keys.get(i).expect("to exist in map_keys").as_ref();
+                        let mut out = vec![];
+                        if let Some(k) = k.strip_prefix("0x") {
+                            let value = hex::decode(k).expect("str must be hex encoded");
+                            let _ = to_bytes_with_info(&mut out, &value, Some((portable_reg, type_id)));
+                        } else {
+                            let _ = to_bytes_with_info(&mut out, &k, Some((portable_reg, type_id)));
+                        }
+                        out
+                    })
+                    .collect();
+
+            let hasher = hashers.first().expect("hasher not found");
+            let hasher = hasher.borrow();
+            let hashed_value = hash(hasher, &touple_hex);
 
-        let storage_key = StorageKey::new(
-            value_ty_id,
-            hash(&Hasher::Twox128, pallet_item.0),
-            hash(&Hasher::Twox128, pallet_item.1),
-            type_call_ids
-                .into_iter()
-                .enumerate()
-                .map(|(i, type_id)| {
-                    let k = map_keys.get(i);
-                    let hasher = hashers.get(i).expect("hasher not found");
-
-                    if k.is_none() {
-                        return KeyValue::Empty(type_id);
-                    }
-
-                    let k = k.expect("it must exist").as_ref();
-
-                    let hasher = hasher.borrow();
-                    let mut out = vec![];
-
-                    if let Some(k) = k.strip_prefix("0x") {
-                        let value = hex::decode(k).expect("str must be encoded");
-                        let _ = to_bytes_with_info(&mut out, &value, Some((portable_reg, type_id)));
-                    } else {
-                        let _ = to_bytes_with_info(&mut out, &k, Some((portable_reg, type_id)));
-                    }
-
-                    let hash = hash(hasher, &out);
-                    KeyValue::Value((type_id, hash, out, hasher.clone()))
-                })
-                .collect(),
-        );
-
-        Ok(storage_key)
+            let storage_key = StorageKey::new(
+                value_ty_id,
+                hash(&Hasher::Twox128, pallet_item.0),
+                hash(&Hasher::Twox128, pallet_item.1),
+                vec![KeyValue::Value((key_ty_id.expect("must key id must work"), hashed_value, touple_hex, hasher.clone()))]
+            );
+
+            Ok(storage_key)
+        } else {
+            Err(crate::Error::Encode("Wrong number of hashers vs map_keys".into()))
+        }
     }
 }
 
@@ -233,14 +272,17 @@ impl EntryTy for EntryType {
                 hashers,
                 key,
                 value,
-            } => self.build_call(
-                registry,
-                Some(key.id),
-                value.id,
-                (pallet, item),
-                map_keys,
-                hashers,
-            ),
+            } => {
+                log::trace!("key={}, value={}, hasher={:?}", key.id, value.id, hashers);
+                self.build_call(
+                    registry,
+                    Some(key.id),
+                    value.id,
+                    (pallet, item),
+                    map_keys,
+                    hashers,
+                )
+            },
         }
     }
 }

sube/src/rpc.rs

@@ -6,6 +6,7 @@ use serde::Deserialize;
 use crate::meta::{self, Metadata};
 use crate::{prelude::*, StorageChangeSet};
 use crate::{Backend, StorageKey};
+use meta::from_bytes;
 
 pub type RpcResult<T> = Result<T, error::Error>;
 
@@ -118,8 +119,7 @@ impl<R: Rpc> Backend for RpcClient<R> {
             .await
             .map_err(|e| crate::Error::Node(e.to_string()))?;
         let response = hex::decode(&res[2..]).map_err(|_err| crate::Error::StorageKeyNotFound)?;
-        let meta =
-            meta::from_bytes(&mut response.as_slice()).map_err(|_| crate::Error::BadMetadata)?;
+        let meta = from_bytes(&mut response.as_slice()).map_err(|_| crate::Error::BadMetadata)?;
         log::trace!("Metadata {:#?}", meta);
         Ok(meta)
     }