From 255b6c32f84b2f867bbe5f8c38eae684bf9cdf0b Mon Sep 17 00:00:00 2001 From: muzarski Date: Wed, 31 Jan 2024 16:08:40 +0100 Subject: [PATCH 1/7] value: fix Value implementation for BigDecimal Before this commit, we wouldn't check for the value overflow when computing the bigdecimal serialized bytes length. After this commit, we make use of explicit overflow checks before performing the addition. --- scylla-cql/src/frame/value.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scylla-cql/src/frame/value.rs b/scylla-cql/src/frame/value.rs index 3630bfa630..2e51ec257f 100644 --- a/scylla-cql/src/frame/value.rs +++ b/scylla-cql/src/frame/value.rs @@ -886,9 +886,13 @@ impl Value for BigDecimal { let (value, scale) = self.as_bigint_and_exponent(); let serialized = value.to_signed_bytes_be(); - let serialized_len: i32 = serialized.len().try_into().map_err(|_| ValueTooBig)?; - buf.put_i32(serialized_len + 4); + if serialized.len() > (i32::MAX - 4) as usize { + return Err(ValueTooBig); + } + let serialized_len: i32 = serialized.len() as i32 + 4; + + buf.put_i32(serialized_len); buf.put_i32(scale.try_into().map_err(|_| ValueTooBig)?); buf.extend_from_slice(&serialized); From e3aa0099c36e221e63b09b5e39b1471cdc80ba6d Mon Sep 17 00:00:00 2001 From: muzarski Date: Wed, 31 Jan 2024 17:12:34 +0100 Subject: [PATCH 2/7] cargo: bump bigdecimal version to 0.4 --- Cargo.lock.msrv | 14 +++++++++++--- docs/source/quickstart/create-project.md | 2 +- scylla-cql/Cargo.toml | 2 +- scylla-proxy/Cargo.toml | 2 +- scylla/Cargo.toml | 2 +- 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Cargo.lock.msrv b/Cargo.lock.msrv index 84b515fccc..adf4d26446 100644 --- a/Cargo.lock.msrv +++ b/Cargo.lock.msrv @@ -116,11 +116,13 @@ checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bigdecimal" -version = "0.2.2" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1e50562e37200edf7c6c43e54a08e64a5553bfb59d9c297d5572512aa517256" +checksum = "c06619be423ea5bb86c95f087d5707942791a08a85530df0db2209a3ecfb8bc9" dependencies = [ - "num-bigint 0.3.3", + "autocfg", + "libm", + "num-bigint 0.4.4", "num-integer", "num-traits", ] @@ -794,6 +796,12 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libm" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" + [[package]] name = "linux-raw-sys" version = "0.4.7" diff --git a/docs/source/quickstart/create-project.md b/docs/source/quickstart/create-project.md index c6ee6bc949..8bf5af1f03 100644 --- a/docs/source/quickstart/create-project.md +++ b/docs/source/quickstart/create-project.md @@ -12,7 +12,7 @@ scylla = "0.11" tokio = { version = "1.12", features = ["full"] } futures = "0.3.6" uuid = "1.0" -bigdecimal = "0.2.0" +bigdecimal = "0.4" num-bigint = "0.3" tracing = "0.1.36" tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } diff --git a/scylla-cql/Cargo.toml b/scylla-cql/Cargo.toml index 76adedcb91..36f27be714 100644 --- a/scylla-cql/Cargo.toml +++ b/scylla-cql/Cargo.toml @@ -19,7 +19,7 @@ secrecy = { version = "0.7.0", optional = true } snap = "1.0" uuid = "1.0" thiserror = "1.0" -bigdecimal = "0.2.0" +bigdecimal = "0.4" num-bigint-03 = { package = "num-bigint", version = "0.3", optional = true } num-bigint-04 = { package = "num-bigint", version = "0.4", optional = true } chrono = { version = "0.4.27", default-features = false, optional = true } diff --git a/scylla-proxy/Cargo.toml b/scylla-proxy/Cargo.toml index 9d630b1bdd..6b25425d10 100644 --- a/scylla-proxy/Cargo.toml +++ b/scylla-proxy/Cargo.toml @@ -21,7 +21,7 @@ num_enum = "0.5" tokio = { version = "1.12", features = ["net", "time", "io-util", "sync", "rt", "macros", "rt-multi-thread"] } uuid = "1.0" thiserror = "1.0.32" -bigdecimal = "0.2.0" +bigdecimal = "0.4" num-bigint = "0.3" tracing = "0.1.25" chrono = { version = "0.4", default-features = false } diff --git a/scylla/Cargo.toml b/scylla/Cargo.toml index 2d59482296..0c54880e8d 100644 --- a/scylla/Cargo.toml +++ b/scylla/Cargo.toml @@ -38,7 +38,7 @@ uuid = { version = "1.0", features = ["v4"] } rand = "0.8.3" thiserror = "1.0" itertools = "0.11.0" -bigdecimal = "0.2.0" +bigdecimal = "0.4" tracing = "0.1.36" chrono = { version = "0.4.20", default-features = false, features = ["clock"] } openssl = { version = "0.10.32", optional = true } From 652e387d1702b2aa25664419bc9efdd007feb6e2 Mon Sep 17 00:00:00 2001 From: muzarski Date: Fri, 12 Jan 2024 12:20:33 +0100 Subject: [PATCH 3/7] cql: add CqlDecimal type --- scylla-cql/src/frame/value.rs | 77 +++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/scylla-cql/src/frame/value.rs b/scylla-cql/src/frame/value.rs index 2e51ec257f..5558b8bff0 100644 --- a/scylla-cql/src/frame/value.rs +++ b/scylla-cql/src/frame/value.rs @@ -365,6 +365,83 @@ impl std::hash::Hash for CqlVarint { } } +/// Native CQL `decimal` representation. +/// +/// Represented as a pair: +/// - a [`CqlVarint`] value +/// - 32-bit integer which determines the position of the decimal point +/// +/// # DB data format +/// Notice that [constructors](CqlDecimal#impl-CqlDecimal) +/// don't perform any normalization on the provided data. +/// For more details, see [`CqlVarint`] documentation. +#[derive(Clone, PartialEq, Eq, Debug)] +pub struct CqlDecimal { + int_val: CqlVarint, + scale: i32, +} + +/// Constructors +impl CqlDecimal { + /// Creates a [`CqlDecimal`] from an array of bytes + /// representing [`CqlVarint`] and a 32-bit scale. + /// + /// See: disclaimer about [non-normalized values](CqlVarint#db-data-format). + pub fn from_signed_be_bytes_and_exponent(bytes: Vec, scale: i32) -> Self { + Self { + int_val: CqlVarint::from_signed_bytes_be(bytes), + scale, + } + } + + /// Creates a [`CqlDecimal`] from a slice of bytes + /// representing [`CqlVarint`] and a 32-bit scale. + /// + /// See: disclaimer about [non-normalized values](CqlVarint#db-data-format). + pub fn from_signed_be_bytes_slice_and_exponent(bytes: &[u8], scale: i32) -> Self { + Self::from_signed_be_bytes_and_exponent(bytes.to_vec(), scale) + } +} + +/// Conversion to raw bytes +impl CqlDecimal { + /// Returns a slice of bytes in two's complement + /// binary big-endian representation and a scale. + pub fn as_signed_be_bytes_slice_and_exponent(&self) -> (&[u8], i32) { + (self.int_val.as_signed_bytes_be_slice(), self.scale) + } + + /// Converts [`CqlDecimal`] to an array of bytes in two's + /// complement binary big-endian representation and a scale. + pub fn into_signed_be_bytes_and_exponent(self) -> (Vec, i32) { + (self.int_val.into_signed_bytes_be(), self.scale) + } +} + +impl From for BigDecimal { + fn from(value: CqlDecimal) -> Self { + Self::from(( + bigdecimal::num_bigint::BigInt::from_signed_bytes_be( + value.int_val.as_signed_bytes_be_slice(), + ), + value.scale as i64, + )) + } +} + +impl TryFrom for CqlDecimal { + type Error = >::Error; + + fn try_from(value: BigDecimal) -> Result { + let (bigint, scale) = value.into_bigint_and_exponent(); + let bytes = bigint.to_signed_bytes_be(); + Ok(Self::from_signed_be_bytes_and_exponent( + bytes, + scale.try_into()?, + )) + } +} + /// Native CQL date representation that allows for a bigger range of dates (-262145-1-1 to 262143-12-31). /// /// Represented as number of days since -5877641-06-23 i.e. 2^31 days before unix epoch. From 5c6ebe12a58f5bf91351d3cbecc377c6fd1573cd Mon Sep 17 00:00:00 2001 From: muzarski Date: Fri, 12 Jan 2024 13:00:05 +0100 Subject: [PATCH 4/7] cql: wrap CqlDecimal with CqlValue::Decimal --- scylla-cql/src/frame/response/cql_to_rust.rs | 15 ++++++++++--- scylla-cql/src/frame/response/result.rs | 19 ++++++++++------- scylla-cql/src/frame/value.rs | 17 +++++++++++++++ scylla-cql/src/types/serialize/value.rs | 16 ++++++++++++-- scylla/src/utils/pretty.rs | 22 +++++++++++++------- 5 files changed, 69 insertions(+), 20 deletions(-) diff --git a/scylla-cql/src/frame/response/cql_to_rust.rs b/scylla-cql/src/frame/response/cql_to_rust.rs index 9030e8d9e9..7bc4f86e89 100644 --- a/scylla-cql/src/frame/response/cql_to_rust.rs +++ b/scylla-cql/src/frame/response/cql_to_rust.rs @@ -1,6 +1,6 @@ use super::result::{CqlValue, Row}; use crate::frame::value::{ - Counter, CqlDate, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, + Counter, CqlDate, CqlDecimal, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, }; use bigdecimal::BigDecimal; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -136,7 +136,7 @@ impl_from_cql_value_from_method!(Vec, into_blob); // Vec::from_cql impl_from_cql_value_from_method!(Uuid, as_uuid); // Uuid::from_cql impl_from_cql_value_from_method!(CqlTimeuuid, as_timeuuid); // CqlTimeuuid::from_cql -impl_from_cql_value_from_method!(BigDecimal, into_decimal); // BigDecimal::from_cql +impl_from_cql_value_from_method!(CqlDecimal, into_cql_decimal); // CqlDecimal::from_cql impl_from_cql_value_from_method!(CqlDuration, as_cql_duration); // CqlDuration::from_cql impl_from_cql_value_from_method!(CqlDate, as_cql_date); // CqlDate::from_cql impl_from_cql_value_from_method!(CqlTime, as_cql_time); // CqlTime::from_cql @@ -169,6 +169,15 @@ impl FromCqlVal for num_bigint_04::BigInt { } } +impl FromCqlVal for BigDecimal { + fn from_cql(cql_val: CqlValue) -> Result { + match cql_val { + CqlValue::Decimal(cql_decimal) => Ok(cql_decimal.into()), + _ => Err(FromCqlValError::BadCqlType), + } + } +} + #[cfg(feature = "chrono")] impl FromCqlVal for NaiveDate { fn from_cql(cql_val: CqlValue) -> Result { @@ -507,7 +516,7 @@ mod tests { let decimal = BigDecimal::from_str("123.4").unwrap(); assert_eq!( Ok(decimal.clone()), - BigDecimal::from_cql(CqlValue::Decimal(decimal)) + BigDecimal::from_cql(CqlValue::Decimal(decimal.try_into().unwrap())) ); } diff --git a/scylla-cql/src/frame/response/result.rs b/scylla-cql/src/frame/response/result.rs index ffcd35543b..75eb21f9e6 100644 --- a/scylla-cql/src/frame/response/result.rs +++ b/scylla-cql/src/frame/response/result.rs @@ -2,10 +2,9 @@ use crate::cql_to_rust::{FromRow, FromRowError}; use crate::frame::response::event::SchemaChangeEvent; use crate::frame::types::vint_decode; use crate::frame::value::{ - Counter, CqlDate, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, + Counter, CqlDate, CqlDecimal, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, }; use crate::frame::{frame_errors::ParseError, types}; -use bigdecimal::BigDecimal; use byteorder::{BigEndian, ReadBytesExt}; use bytes::{Buf, Bytes}; use std::{ @@ -82,7 +81,7 @@ pub enum CqlValue { Boolean(bool), Blob(Vec), Counter(Counter), - Decimal(BigDecimal), + Decimal(CqlDecimal), /// Days since -5877641-06-23 i.e. 2^31 days before unix epoch /// Can be converted to chrono::NaiveDate (-262145-1-1 to 262143-12-31) using as_date Date(CqlDate), @@ -371,7 +370,7 @@ impl CqlValue { } } - pub fn into_decimal(self) -> Option { + pub fn into_cql_decimal(self) -> Option { match self { Self::Decimal(i) => Some(i), _ => None, @@ -671,9 +670,10 @@ pub fn deser_cql_value(typ: &ColumnType, buf: &mut &[u8]) -> StdResult()?)) } Decimal => { - let scale = types::read_int(buf)? as i64; - let int_value = bigdecimal::num_bigint::BigInt::from_signed_bytes_be(buf); - let big_decimal: BigDecimal = BigDecimal::from((int_value, scale)); + let scale = types::read_int(buf)?; + let bytes = buf.to_vec(); + let big_decimal: CqlDecimal = + CqlDecimal::from_signed_be_bytes_and_exponent(bytes, scale); CqlValue::Decimal(big_decimal) } @@ -1139,7 +1139,10 @@ mod tests { for t in tests.iter() { let value = super::deser_cql_value(&ColumnType::Decimal, &mut &*t.encoding).unwrap(); - assert_eq!(CqlValue::Decimal(t.value.clone()), value); + assert_eq!( + CqlValue::Decimal(t.value.clone().try_into().unwrap()), + value + ); } } diff --git a/scylla-cql/src/frame/value.rs b/scylla-cql/src/frame/value.rs index 5558b8bff0..fe6f852694 100644 --- a/scylla-cql/src/frame/value.rs +++ b/scylla-cql/src/frame/value.rs @@ -958,6 +958,23 @@ impl Value for i64 { } } +impl Value for CqlDecimal { + fn serialize(&self, buf: &mut Vec) -> Result<(), ValueTooBig> { + let (bytes, scale) = self.as_signed_be_bytes_slice_and_exponent(); + + if bytes.len() > (i32::MAX - 4) as usize { + return Err(ValueTooBig); + } + let serialized_len: i32 = bytes.len() as i32 + 4; + + buf.put_i32(serialized_len); + buf.put_i32(scale); + buf.extend_from_slice(bytes); + + Ok(()) + } +} + impl Value for BigDecimal { fn serialize(&self, buf: &mut Vec) -> Result<(), ValueTooBig> { let (value, scale) = self.as_bigint_and_exponent(); diff --git a/scylla-cql/src/types/serialize/value.rs b/scylla-cql/src/types/serialize/value.rs index a50dc23d21..58c656cb73 100644 --- a/scylla-cql/src/types/serialize/value.rs +++ b/scylla-cql/src/types/serialize/value.rs @@ -19,8 +19,8 @@ use secrecy::{ExposeSecret, Secret, Zeroize}; use crate::frame::response::result::{ColumnType, CqlValue}; use crate::frame::types::vint_encode; use crate::frame::value::{ - Counter, CqlDate, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, MaybeUnset, - Unset, Value, + Counter, CqlDate, CqlDecimal, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, + MaybeUnset, Unset, Value, }; #[cfg(feature = "chrono")] @@ -115,6 +115,18 @@ impl SerializeCql for i64 { writer.set_value(me.to_be_bytes().as_slice()).unwrap() }); } +impl SerializeCql for CqlDecimal { + impl_serialize_via_writer!(|me, typ, writer| { + exact_type_check!(typ, Decimal); + let mut builder = writer.into_value_builder(); + let (bytes, scale) = me.as_signed_be_bytes_slice_and_exponent(); + builder.append_bytes(&scale.to_be_bytes()); + builder.append_bytes(bytes); + builder + .finish() + .map_err(|_| mk_ser_err::(typ, BuiltinSerializationErrorKind::SizeOverflow))? + }); +} impl SerializeCql for BigDecimal { impl_serialize_via_writer!(|me, typ, writer| { exact_type_check!(typ, Decimal); diff --git a/scylla/src/utils/pretty.rs b/scylla/src/utils/pretty.rs index 56d0c78514..5040ddb507 100644 --- a/scylla/src/utils/pretty.rs +++ b/scylla/src/utils/pretty.rs @@ -42,7 +42,15 @@ where CqlValue::Text(t) => write!(f, "{}", CqlStringLiteralDisplayer(t))?, CqlValue::Blob(b) => write!(f, "0x{:x}", HexBytes(b))?, CqlValue::Empty => write!(f, "0x")?, - CqlValue::Decimal(d) => write!(f, "{}", d)?, + CqlValue::Decimal(d) => { + let (bytes, scale) = d.as_signed_be_bytes_slice_and_exponent(); + write!( + f, + "blobAsDecimal(0x{:x}{:x})", + HexBytes(&scale.to_be_bytes()), + HexBytes(bytes) + )? + } CqlValue::Float(fl) => write!(f, "{}", fl)?, CqlValue::Double(d) => write!(f, "{}", d)?, CqlValue::Boolean(b) => write!(f, "{}", b)?, @@ -211,13 +219,10 @@ where #[cfg(test)] mod tests { - use std::str::FromStr; - - use bigdecimal::BigDecimal; use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; use scylla_cql::frame::response::result::CqlValue; - use scylla_cql::frame::value::{CqlDate, CqlDuration, CqlTime, CqlTimestamp}; + use scylla_cql::frame::value::{CqlDate, CqlDecimal, CqlDuration, CqlTime, CqlTimestamp}; use crate::utils::pretty::CqlValueDisplayer; @@ -231,9 +236,12 @@ mod tests { assert_eq!( format!( "{}", - CqlValueDisplayer(CqlValue::Decimal(BigDecimal::from_str("123.456").unwrap())) + // 123.456 + CqlValueDisplayer(CqlValue::Decimal( + CqlDecimal::from_signed_be_bytes_and_exponent(vec![0x01, 0xE2, 0x40], 3) + )) ), - "123.456" + "blobAsDecimal(0x0000000301e240)" ); assert_eq!( format!("{}", CqlValueDisplayer(CqlValue::Float(12.75))), From 176218ef118180e3957db5d60ebf1b5d2fab29a2 Mon Sep 17 00:00:00 2001 From: muzarski Date: Fri, 26 Jan 2024 16:44:16 +0100 Subject: [PATCH 5/7] cargo: add bigdecimal-04 crate feature --- examples/Cargo.toml | 2 +- scylla-cql/Cargo.toml | 5 +++-- scylla-cql/src/frame/response/cql_to_rust.rs | 10 +++++----- scylla-cql/src/frame/response/result.rs | 3 ++- scylla-cql/src/frame/value.rs | 20 ++++++++++++++------ scylla-cql/src/frame/value_tests.rs | 13 +++++++++---- scylla-cql/src/types/serialize/value.rs | 13 +++++++++---- scylla/Cargo.toml | 5 +++-- scylla/src/transport/cql_types_test.rs | 4 ++-- 9 files changed, 48 insertions(+), 27 deletions(-) diff --git a/examples/Cargo.toml b/examples/Cargo.toml index 66e1e2d47e..467963f93f 100644 --- a/examples/Cargo.toml +++ b/examples/Cargo.toml @@ -10,7 +10,7 @@ futures = "0.3.6" openssl = "0.10.32" rustyline = "9" rustyline-derive = "0.6" -scylla = {path = "../scylla", features = ["ssl", "cloud", "chrono", "time", "num-bigint-03", "num-bigint-04"]} +scylla = {path = "../scylla", features = ["ssl", "cloud", "chrono", "time", "num-bigint-03", "num-bigint-04", "bigdecimal-04"]} tokio = {version = "1.1.0", features = ["full"]} tracing = "0.1.25" tracing-subscriber = { version = "0.3.14", features = ["env-filter"] } diff --git a/scylla-cql/Cargo.toml b/scylla-cql/Cargo.toml index 36f27be714..6cf72b2a9b 100644 --- a/scylla-cql/Cargo.toml +++ b/scylla-cql/Cargo.toml @@ -19,9 +19,9 @@ secrecy = { version = "0.7.0", optional = true } snap = "1.0" uuid = "1.0" thiserror = "1.0" -bigdecimal = "0.4" num-bigint-03 = { package = "num-bigint", version = "0.3", optional = true } num-bigint-04 = { package = "num-bigint", version = "0.4", optional = true } +bigdecimal-04 = { package = "bigdecimal", version = "0.4", optional = true } chrono = { version = "0.4.27", default-features = false, optional = true } lz4_flex = { version = "0.11.1" } async-trait = "0.1.57" @@ -43,4 +43,5 @@ time = ["dep:time"] chrono = ["dep:chrono"] num-bigint-03 = ["dep:num-bigint-03"] num-bigint-04 = ["dep:num-bigint-04"] -full-serialization = ["chrono", "time", "secret", "num-bigint-03", "num-bigint-04"] +bigdecimal-04 = ["dep:bigdecimal-04"] +full-serialization = ["chrono", "time", "secret", "num-bigint-03", "num-bigint-04", "bigdecimal-04"] diff --git a/scylla-cql/src/frame/response/cql_to_rust.rs b/scylla-cql/src/frame/response/cql_to_rust.rs index 7bc4f86e89..f40f2346fb 100644 --- a/scylla-cql/src/frame/response/cql_to_rust.rs +++ b/scylla-cql/src/frame/response/cql_to_rust.rs @@ -2,7 +2,6 @@ use super::result::{CqlValue, Row}; use crate::frame::value::{ Counter, CqlDate, CqlDecimal, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid, CqlVarint, }; -use bigdecimal::BigDecimal; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::hash::{BuildHasher, Hash}; use std::net::IpAddr; @@ -169,7 +168,8 @@ impl FromCqlVal for num_bigint_04::BigInt { } } -impl FromCqlVal for BigDecimal { +#[cfg(feature = "bigdecimal-04")] +impl FromCqlVal for bigdecimal_04::BigDecimal { fn from_cql(cql_val: CqlValue) -> Result { match cql_val { CqlValue::Decimal(cql_decimal) => Ok(cql_decimal.into()), @@ -423,7 +423,6 @@ mod tests { use crate as scylla; use crate::frame::value::{Counter, CqlDate, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid}; use crate::macros::FromRow; - use bigdecimal::BigDecimal; use std::collections::HashSet; use std::net::{IpAddr, Ipv4Addr}; use std::str::FromStr; @@ -511,12 +510,13 @@ mod tests { ); } + #[cfg(feature = "bigdecimal-04")] #[test] fn decimal_from_cql() { - let decimal = BigDecimal::from_str("123.4").unwrap(); + let decimal = bigdecimal_04::BigDecimal::from_str("123.4").unwrap(); assert_eq!( Ok(decimal.clone()), - BigDecimal::from_cql(CqlValue::Decimal(decimal.try_into().unwrap())) + bigdecimal_04::BigDecimal::from_cql(CqlValue::Decimal(decimal.try_into().unwrap())) ); } diff --git a/scylla-cql/src/frame/response/result.rs b/scylla-cql/src/frame/response/result.rs index 75eb21f9e6..745b925c00 100644 --- a/scylla-cql/src/frame/response/result.rs +++ b/scylla-cql/src/frame/response/result.rs @@ -967,7 +967,6 @@ pub fn deserialize(buf: &mut &[u8]) -> StdResult { mod tests { use crate as scylla; use crate::frame::value::{Counter, CqlDate, CqlDuration, CqlTime, CqlTimestamp, CqlTimeuuid}; - use bigdecimal::BigDecimal; use scylla::frame::response::result::{ColumnType, CqlValue}; use std::str::FromStr; use uuid::Uuid; @@ -1111,8 +1110,10 @@ mod tests { } } + #[cfg(feature = "bigdecimal-04")] #[test] fn test_decimal() { + use bigdecimal_04::BigDecimal; struct Test<'a> { value: BigDecimal, encoding: &'a [u8], diff --git a/scylla-cql/src/frame/value.rs b/scylla-cql/src/frame/value.rs index fe6f852694..991d9ad59d 100644 --- a/scylla-cql/src/frame/value.rs +++ b/scylla-cql/src/frame/value.rs @@ -1,6 +1,5 @@ use crate::frame::frame_errors::ParseError; use crate::frame::types; -use bigdecimal::BigDecimal; use bytes::BufMut; use std::borrow::Cow; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -371,6 +370,12 @@ impl std::hash::Hash for CqlVarint { /// - a [`CqlVarint`] value /// - 32-bit integer which determines the position of the decimal point /// +/// The type is not very useful in most use cases. +/// However, users can make use of more complex types +/// such as `bigdecimal::BigDecimal` (v0.4). +/// The library support (e.g. conversion from [`CqlValue`]) for the type is +/// enabled via `bigdecimal-04` crate feature. +/// /// # DB data format /// Notice that [constructors](CqlDecimal#impl-CqlDecimal) /// don't perform any normalization on the provided data. @@ -418,10 +423,11 @@ impl CqlDecimal { } } -impl From for BigDecimal { +#[cfg(feature = "bigdecimal-04")] +impl From for bigdecimal_04::BigDecimal { fn from(value: CqlDecimal) -> Self { Self::from(( - bigdecimal::num_bigint::BigInt::from_signed_bytes_be( + bigdecimal_04::num_bigint::BigInt::from_signed_bytes_be( value.int_val.as_signed_bytes_be_slice(), ), value.scale as i64, @@ -429,10 +435,11 @@ impl From for BigDecimal { } } -impl TryFrom for CqlDecimal { +#[cfg(feature = "bigdecimal-04")] +impl TryFrom for CqlDecimal { type Error = >::Error; - fn try_from(value: BigDecimal) -> Result { + fn try_from(value: bigdecimal_04::BigDecimal) -> Result { let (bigint, scale) = value.into_bigint_and_exponent(); let bytes = bigint.to_signed_bytes_be(); Ok(Self::from_signed_be_bytes_and_exponent( @@ -975,7 +982,8 @@ impl Value for CqlDecimal { } } -impl Value for BigDecimal { +#[cfg(feature = "bigdecimal-04")] +impl Value for bigdecimal_04::BigDecimal { fn serialize(&self, buf: &mut Vec) -> Result<(), ValueTooBig> { let (value, scale) = self.as_bigint_and_exponent(); diff --git a/scylla-cql/src/frame/value_tests.rs b/scylla-cql/src/frame/value_tests.rs index 978c7a6db0..debf979249 100644 --- a/scylla-cql/src/frame/value_tests.rs +++ b/scylla-cql/src/frame/value_tests.rs @@ -10,7 +10,6 @@ use super::value::{ CqlDate, CqlDuration, CqlTime, CqlTimestamp, LegacyBatchValues, LegacySerializedValues, MaybeUnset, SerializeValuesError, Unset, Value, ValueList, ValueTooBig, }; -use bigdecimal::BigDecimal; use bytes::BufMut; use std::collections::hash_map::DefaultHasher; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; @@ -147,6 +146,11 @@ fn cql_varint_serialization() { } } +#[cfg(any( + feature = "num-bigint-03", + feature = "num-bigint-04", + feature = "bigdecimal-04" +))] fn varint_test_cases_from_spec() -> Vec<(i64, Vec)> { vec![ (0, vec![0x00]), @@ -191,8 +195,9 @@ fn bigint04_serialization() { generic_num_bigint_serialization::() } +#[cfg(feature = "bigdecimal-04")] #[test] -fn bigdecimal_serialization() { +fn bigdecimal04_serialization() { // Bigint cases let cases_from_the_spec: &[(i64, Vec)] = &varint_test_cases_from_spec(); @@ -205,8 +210,8 @@ fn bigdecimal_serialization() { .chain(serialized_digits) .cloned() .collect::>(); - let digits = bigdecimal::num_bigint::BigInt::from(*digits); - let x = BigDecimal::new(digits, exponent as i64); + let digits = bigdecimal_04::num_bigint::BigInt::from(*digits); + let x = bigdecimal_04::BigDecimal::new(digits, exponent as i64); assert_eq!(serialized(x, ColumnType::Decimal), repr); } } diff --git a/scylla-cql/src/types/serialize/value.rs b/scylla-cql/src/types/serialize/value.rs index 58c656cb73..39403f3c48 100644 --- a/scylla-cql/src/types/serialize/value.rs +++ b/scylla-cql/src/types/serialize/value.rs @@ -6,7 +6,6 @@ use std::hash::BuildHasher; use std::net::IpAddr; use std::sync::Arc; -use bigdecimal::BigDecimal; use thiserror::Error; use uuid::Uuid; @@ -127,7 +126,8 @@ impl SerializeCql for CqlDecimal { .map_err(|_| mk_ser_err::(typ, BuiltinSerializationErrorKind::SizeOverflow))? }); } -impl SerializeCql for BigDecimal { +#[cfg(feature = "bigdecimal-04")] +impl SerializeCql for bigdecimal_04::BigDecimal { impl_serialize_via_writer!(|me, typ, writer| { exact_type_check!(typ, Decimal); let mut builder = writer.into_value_builder(); @@ -1536,8 +1536,6 @@ mod tests { }; use crate::types::serialize::{CellWriter, SerializationError}; - use bigdecimal::num_bigint::BigInt; - use bigdecimal::BigDecimal; use scylla_macros::SerializeCql; use super::{SerializeCql, UdtSerializationErrorKind, UdtTypeCheckErrorKind}; @@ -1653,6 +1651,13 @@ mod tests { // We'll skip testing for SizeOverflow as this would require producing // a value which is at least 2GB in size. + } + + #[cfg(feature = "bigdecimal-04")] + #[test] + fn test_native_errors_bigdecimal_04() { + use bigdecimal_04::num_bigint::BigInt; + use bigdecimal_04::BigDecimal; // Value overflow (type out of representable range) let v = BigDecimal::new(BigInt::from(123), 1i64 << 40); diff --git a/scylla/Cargo.toml b/scylla/Cargo.toml index 0c54880e8d..9f8654d985 100644 --- a/scylla/Cargo.toml +++ b/scylla/Cargo.toml @@ -22,7 +22,8 @@ chrono = ["scylla-cql/chrono"] time = ["scylla-cql/time"] num-bigint-03 = ["scylla-cql/num-bigint-03"] num-bigint-04 = ["scylla-cql/num-bigint-04"] -full-serialization = ["chrono", "time", "secret", "num-bigint-03", "num-bigint-04"] +bigdecimal-04 = ["scylla-cql/bigdecimal-04"] +full-serialization = ["chrono", "time", "secret", "num-bigint-03", "num-bigint-04", "bigdecimal-04"] [dependencies] scylla-macros = { version = "0.3.0", path = "../scylla-macros" } @@ -38,7 +39,6 @@ uuid = { version = "1.0", features = ["v4"] } rand = "0.8.3" thiserror = "1.0" itertools = "0.11.0" -bigdecimal = "0.4" tracing = "0.1.36" chrono = { version = "0.4.20", default-features = false, features = ["clock"] } openssl = { version = "0.10.32", optional = true } @@ -60,6 +60,7 @@ socket2 = { version = "0.5.3", features = ["all"] } [dev-dependencies] num-bigint-03 = { package = "num-bigint", version = "0.3" } num-bigint-04 = { package = "num-bigint", version = "0.4" } +bigdecimal-04 = { package = "bigdecimal", version = "0.4" } scylla-proxy = { version = "0.0.3", path = "../scylla-proxy" } ntest = "0.9.0" criterion = "0.4" # Note: v0.5 needs at least rust 1.70.0 diff --git a/scylla/src/transport/cql_types_test.rs b/scylla/src/transport/cql_types_test.rs index aac8561d8c..4dda187328 100644 --- a/scylla/src/transport/cql_types_test.rs +++ b/scylla/src/transport/cql_types_test.rs @@ -7,7 +7,6 @@ use crate::test_utils::create_new_session_builder; use crate::transport::session::IntoTypedRows; use crate::transport::session::Session; use crate::utils::test_utils::unique_keyspace_name; -use bigdecimal::BigDecimal; use itertools::Itertools; use scylla_cql::frame::value::{CqlTimeuuid, CqlVarint}; use scylla_cql::types::serialize::value::SerializeCql; @@ -218,6 +217,7 @@ async fn test_cql_varint() { } } +#[cfg(feature = "bigdecimal-04")] #[tokio::test] async fn test_decimal() { let tests = [ @@ -229,7 +229,7 @@ async fn test_decimal() { "-123456789012345678901234567890.1234567890", ]; - run_tests::(&tests, "decimal").await; + run_tests::(&tests, "decimal").await; } #[tokio::test] From 145e3bb4efe69310184bb39bfc36c4a372909357 Mon Sep 17 00:00:00 2001 From: muzarski Date: Fri, 26 Jan 2024 17:26:03 +0100 Subject: [PATCH 6/7] CI: add cargo checks with bigdecimal features --- .github/workflows/rust.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9415c63b0f..d11f5bbab1 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -44,6 +44,8 @@ jobs: run: cargo check --all-targets --manifest-path "scylla/Cargo.toml" --features "num-bigint-03" - name: Cargo check with num-bigint-04 feature run: cargo check --all-targets --manifest-path "scylla/Cargo.toml" --features "num-bigint-04" + - name: Cargo check with bigdecimal-04 feature + run: cargo check --all-targets --manifest-path "scylla/Cargo.toml" --features "bigdecimal-04" - name: Build scylla-cql run: cargo build --verbose --all-targets --manifest-path "scylla-cql/Cargo.toml" --features "full-serialization" - name: Build From 10bd78559db3ac1aed1ec076f79d00f875a3dd73 Mon Sep 17 00:00:00 2001 From: muzarski Date: Fri, 26 Jan 2024 17:58:03 +0100 Subject: [PATCH 7/7] docs: add info about CqlDecimal --- docs/source/data-types/data-types.md | 2 +- docs/source/data-types/decimal.md | 36 +++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/docs/source/data-types/data-types.md b/docs/source/data-types/data-types.md index e03c82233e..0d9696e765 100644 --- a/docs/source/data-types/data-types.md +++ b/docs/source/data-types/data-types.md @@ -26,7 +26,7 @@ Database types and their Rust equivalents: * `Time` <----> `value::CqlTime`, `chrono::NaiveTime`, `time::Time` * `Timestamp` <----> `value::CqlTimestamp`, `chrono::DateTime`, `time::OffsetDateTime` * `Duration` <----> `value::CqlDuration` -* `Decimal` <----> `bigdecimal::Decimal` +* `Decimal` <----> `value::CqlDecimal`, `bigdecimal::Decimal` * `Varint` <----> `value::CqlVarint`, `num_bigint::BigInt` (v0.3 and v0.4) * `List` <----> `Vec` * `Set` <----> `Vec` diff --git a/docs/source/data-types/decimal.md b/docs/source/data-types/decimal.md index d3d7b45feb..74926c8383 100644 --- a/docs/source/data-types/decimal.md +++ b/docs/source/data-types/decimal.md @@ -1,5 +1,39 @@ # Decimal -`Decimal` is represented as [`bigdecimal::BigDecimal`](https://docs.rs/bigdecimal/0.2.0/bigdecimal/struct.BigDecimal.html) +`Decimal` is represented as `value::CqlDecimal` or [`bigdecimal::BigDecimal`](https://docs.rs/bigdecimal/latest/bigdecimal/struct.BigDecimal.html) + +## value::CqlDecimal + +Without any feature flags, the user can interact with `decimal` type by making use of `value::CqlDecimal` which is a very simple wrapper representing the value as signed binary number in big-endian order with a 32-bit scale. + +```rust +# extern crate scylla; +# use scylla::Session; +# use std::error::Error; +# async fn check_only_compiles(session: &Session) -> Result<(), Box> { +use scylla::IntoTypedRows; +use scylla::frame::value::CqlDecimal; +use std::str::FromStr; + +// Insert a decimal (123.456) into the table +let to_insert: CqlDecimal = + CqlDecimal::from_signed_be_bytes_and_exponent(vec![0x01, 0xE2, 0x40], 3); +session + .query("INSERT INTO keyspace.table (a) VALUES(?)", (to_insert,)) + .await?; + +// Read a decimal from the table +if let Some(rows) = session.query("SELECT a FROM keyspace.table", &[]).await?.rows { + for row in rows.into_typed::<(CqlDecimal,)>() { + let (decimal_value,): (CqlDecimal,) = row?; + } +} +# Ok(()) +# } +``` + +## bigdecimal::BigDecimal + +To make use of `bigdecimal::Bigdecimal` type, user should enable `bigdecimal-04` crate feature. ```rust # extern crate scylla;