From 425e5f59be4e5c99a215879c0b7ddd8feac44ce6 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Thu, 25 Jul 2024 17:49:05 +0800 Subject: [PATCH 1/8] add benchmark --- src/common/Cargo.toml | 4 + .../bench_column_aware_row_encoding.rs | 91 +++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 src/common/benches/bench_column_aware_row_encoding.rs diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index 86e229ddb7b9..a117dce645ae 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -165,6 +165,10 @@ harness = false name = "bench_hash_key_encoding" harness = false +[[bench]] +name = "bench_column_aware_row_encoding" +harness = false + [[bench]] name = "bench_data_chunk_encoding" harness = false diff --git a/src/common/benches/bench_column_aware_row_encoding.rs b/src/common/benches/bench_column_aware_row_encoding.rs new file mode 100644 index 000000000000..91a78ecec7d2 --- /dev/null +++ b/src/common/benches/bench_column_aware_row_encoding.rs @@ -0,0 +1,91 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use criterion::{black_box, criterion_group, criterion_main, Criterion}; +use rand::{Rng, SeedableRng}; +use risingwave_common::catalog::ColumnId; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{DataType, Date, ScalarImpl}; +use risingwave_common::util::value_encoding::column_aware_row_encoding::*; +use risingwave_common::util::value_encoding::*; + +fn bench_column_aware_encoding(c: &mut Criterion) { + let mut rng = rand::rngs::StdRng::seed_from_u64(42); + + // The schema is inspired by the TPC-H lineitem table + let data_types = Arc::new([ + DataType::Int64, + DataType::Int64, + DataType::Int64, + DataType::Int32, + DataType::Decimal, + DataType::Decimal, + DataType::Decimal, + DataType::Decimal, + DataType::Varchar, + DataType::Varchar, + DataType::Date, + DataType::Date, + DataType::Date, + DataType::Varchar, + DataType::Varchar, + DataType::Varchar, + ]); + let row = OwnedRow::new(vec![ + Some(ScalarImpl::Int64(rng.gen())), + Some(ScalarImpl::Int64(rng.gen())), + Some(ScalarImpl::Int64(rng.gen())), + Some(ScalarImpl::Int32(rng.gen())), + Some(ScalarImpl::Decimal("1.0".parse().unwrap())), + Some(ScalarImpl::Decimal("114.514".parse().unwrap())), + None, + Some(ScalarImpl::Decimal("0.08".parse().unwrap())), + Some(ScalarImpl::Utf8("A".into())), + Some(ScalarImpl::Utf8("B".into())), + Some(ScalarImpl::Date(Date::from_ymd_uncheck(2024, 7, 1))), + Some(ScalarImpl::Date(Date::from_ymd_uncheck(2024, 7, 2))), + Some(ScalarImpl::Date(Date::from_ymd_uncheck(2024, 7, 3))), + Some(ScalarImpl::Utf8("D".into())), + None, + Some(ScalarImpl::Utf8("No comments".into())), + ]); + + let column_ids = (1..=data_types.len()) + .map(|i| ColumnId::from(i as i32)) + .collect::>(); + + c.bench_function("bench_column_aware_row_encoding_encode", |b| { + let serializer = Serializer::new(&column_ids[..]); + b.iter(|| { + black_box(serializer.serialize(&row)); + }); + }); + + let serializer = Serializer::new(&column_ids[..]); + let encoded = serializer.serialize(&row); + + c.bench_function("bench_column_aware_row_encoding_decode", |b| { + let deserializer = + Deserializer::new(&column_ids[..], data_types.clone(), std::iter::empty()); + b.iter(|| { + let result = deserializer.deserialize(&encoded).unwrap(); + black_box(result); + }); + }); +} + +criterion_group!(benches, bench_column_aware_encoding); +criterion_main!(benches); From 8491e6deabef15693401e05a796435a32807bb77 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Thu, 25 Jul 2024 17:54:18 +0800 Subject: [PATCH 2/8] improve performance --- .../column_aware_row_encoding.rs | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs index 1bfbc0c92641..db9c8aedf80e 100644 --- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs +++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs @@ -19,7 +19,7 @@ //! We have a `Serializer` and a `Deserializer` for each schema of `Row`, which can be reused //! until schema changes -use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use bitflags::bitflags; @@ -176,7 +176,7 @@ impl ValueRowSerializer for Serializer { /// Should non-null default values be specified, a new field could be added to Deserializer #[derive(Clone)] pub struct Deserializer { - needed_column_ids: BTreeMap, + needed_column_ids: HashMap, schema: Arc<[DataType]>, default_column_values: Vec<(usize, Datum)>, } @@ -193,7 +193,7 @@ impl Deserializer { .iter() .enumerate() .map(|(i, c)| (c.get_id(), i)) - .collect::>(), + .collect::>(), schema, default_column_values: column_with_default.collect(), } @@ -214,12 +214,16 @@ impl ValueRowDeserializer for Deserializer { let data_start_idx = offsets_start_idx + datum_num * offset_bytes; let offsets = &encoded_bytes[offsets_start_idx..data_start_idx]; let data = &encoded_bytes[data_start_idx..]; - let mut datums: Vec> = vec![None; self.schema.len()]; - let mut contained_indices = BTreeSet::new(); + + // initialize datums with default values + let mut datums: Vec = vec![None; self.schema.len()]; + for (i, datum) in &self.default_column_values { + datums[*i] = datum.clone(); + } + for i in 0..datum_num { let this_id = encoded_bytes.get_i32_le(); if let Some(&decoded_idx) = self.needed_column_ids.get(&this_id) { - contained_indices.insert(decoded_idx); let this_offset_start_idx = i * offset_bytes; let mut this_offset_slice = &offsets[this_offset_start_idx..(this_offset_start_idx + offset_bytes)]; @@ -246,15 +250,10 @@ impl ValueRowDeserializer for Deserializer { &mut data_slice, )?) }; - datums[decoded_idx] = Some(data); - } - } - for (id, datum) in &self.default_column_values { - if !contained_indices.contains(id) { - datums[*id].get_or_insert(datum.clone()); + datums[decoded_idx] = data; } } - Ok(datums.into_iter().map(|d| d.unwrap_or(None)).collect()) + Ok(datums) } } From f12a6b0572e0e1db3a513032837ccfb6d5f1c090 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Thu, 25 Jul 2024 19:49:19 +0800 Subject: [PATCH 3/8] use sorted algorithm & add assertion --- .../column_aware_row_encoding.rs | 50 +++++++++---------- 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs index db9c8aedf80e..b199f84482c8 100644 --- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs +++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs @@ -19,7 +19,7 @@ //! We have a `Serializer` and a `Deserializer` for each schema of `Row`, which can be reused //! until schema changes -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::sync::Arc; use bitflags::bitflags; @@ -27,15 +27,6 @@ use bitflags::bitflags; use super::*; use crate::catalog::ColumnId; -// deprecated design of have a Width to represent number of datum -// may be considered should `ColumnId` representation be optimized -// #[derive(Clone, Copy)] -// enum Width { -// Mid(u8), -// Large(u16), -// Extra(u32), -// } - bitflags! { #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct Flag: u8 { @@ -46,6 +37,15 @@ bitflags! { } } +fn column_ids_are_sorted(column_ids: &[ColumnId]) -> bool { + for i in 1..column_ids.len() { + if column_ids[i - 1] >= column_ids[i] { + return false; + } + } + return true; +} + /// `RowEncoding` holds row-specific information for Column-Aware Encoding struct RowEncoding { flag: Flag, @@ -136,6 +136,7 @@ pub struct Serializer { impl Serializer { /// Create a new `Serializer` with current `column_ids` pub fn new(column_ids: &[ColumnId]) -> Self { + assert!(column_ids_are_sorted(column_ids)); // currently we hard-code ColumnId as i32 let mut encoded_column_ids = Vec::with_capacity(column_ids.len() * 4); for id in column_ids { @@ -176,7 +177,7 @@ impl ValueRowSerializer for Serializer { /// Should non-null default values be specified, a new field could be added to Deserializer #[derive(Clone)] pub struct Deserializer { - needed_column_ids: HashMap, + required_column_ids: Vec, schema: Arc<[DataType]>, default_column_values: Vec<(usize, Datum)>, } @@ -188,12 +189,9 @@ impl Deserializer { column_with_default: impl Iterator, ) -> Self { assert_eq!(column_ids.len(), schema.len()); + assert!(column_ids_are_sorted(&column_ids)); Self { - needed_column_ids: column_ids - .iter() - .enumerate() - .map(|(i, c)| (c.get_id(), i)) - .collect::>(), + required_column_ids: column_ids.iter().map(|id| id.get_id()).collect(), schema, default_column_values: column_with_default.collect(), } @@ -221,9 +219,15 @@ impl ValueRowDeserializer for Deserializer { datums[*i] = datum.clone(); } + // The algorithm here leverages the fact that both `required_column_ids` and the encoded column ids are sorted + // We use two pointers (i and j) to iterate through the two arrays, and only deserialize the columns that are required + let mut j = 0usize; // index of required_column_ids for i in 0..datum_num { let this_id = encoded_bytes.get_i32_le(); - if let Some(&decoded_idx) = self.needed_column_ids.get(&this_id) { + while self.required_column_ids[j] < this_id { + j += 1; // leave the default value as is + } + if self.required_column_ids[j] == this_id { let this_offset_start_idx = i * offset_bytes; let mut this_offset_slice = &offsets[this_offset_start_idx..(this_offset_start_idx + offset_bytes)]; @@ -236,21 +240,15 @@ impl ValueRowDeserializer for Deserializer { None } else { let mut data_slice = &data[this_offset..next_offset]; - Some(deserialize_value( - &self.schema[decoded_idx], - &mut data_slice, - )?) + Some(deserialize_value(&self.schema[j], &mut data_slice)?) } } else if this_offset == data.len() { None } else { let mut data_slice = &data[this_offset..]; - Some(deserialize_value( - &self.schema[decoded_idx], - &mut data_slice, - )?) + Some(deserialize_value(&self.schema[j], &mut data_slice)?) }; - datums[decoded_idx] = data; + datums[j] = data; } } Ok(datums) From c8ab9dc03385efbda9166386e59de2b3e6de5b25 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Fri, 26 Jul 2024 10:16:24 +0800 Subject: [PATCH 4/8] cargo clippy --- .../src/util/value_encoding/column_aware_row_encoding.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs index b199f84482c8..099fa3a8a9e7 100644 --- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs +++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs @@ -43,7 +43,7 @@ fn column_ids_are_sorted(column_ids: &[ColumnId]) -> bool { return false; } } - return true; + true } /// `RowEncoding` holds row-specific information for Column-Aware Encoding @@ -189,7 +189,7 @@ impl Deserializer { column_with_default: impl Iterator, ) -> Self { assert_eq!(column_ids.len(), schema.len()); - assert!(column_ids_are_sorted(&column_ids)); + assert!(column_ids_are_sorted(column_ids)); Self { required_column_ids: column_ids.iter().map(|id| id.get_id()).collect(), schema, @@ -216,7 +216,7 @@ impl ValueRowDeserializer for Deserializer { // initialize datums with default values let mut datums: Vec = vec![None; self.schema.len()]; for (i, datum) in &self.default_column_values { - datums[*i] = datum.clone(); + datums[*i].clone_from(datum); } // The algorithm here leverages the fact that both `required_column_ids` and the encoded column ids are sorted From 98a452da6fb6da52a6d03e960fb6b5cc3f285991 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Fri, 26 Jul 2024 14:37:32 +0800 Subject: [PATCH 5/8] revert "use sorted algorithm & add assertion" --- .../column_aware_row_encoding.rs | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs index 099fa3a8a9e7..23b6167cdca9 100644 --- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs +++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs @@ -19,7 +19,7 @@ //! We have a `Serializer` and a `Deserializer` for each schema of `Row`, which can be reused //! until schema changes -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use bitflags::bitflags; @@ -37,15 +37,6 @@ bitflags! { } } -fn column_ids_are_sorted(column_ids: &[ColumnId]) -> bool { - for i in 1..column_ids.len() { - if column_ids[i - 1] >= column_ids[i] { - return false; - } - } - true -} - /// `RowEncoding` holds row-specific information for Column-Aware Encoding struct RowEncoding { flag: Flag, @@ -136,7 +127,6 @@ pub struct Serializer { impl Serializer { /// Create a new `Serializer` with current `column_ids` pub fn new(column_ids: &[ColumnId]) -> Self { - assert!(column_ids_are_sorted(column_ids)); // currently we hard-code ColumnId as i32 let mut encoded_column_ids = Vec::with_capacity(column_ids.len() * 4); for id in column_ids { @@ -177,7 +167,7 @@ impl ValueRowSerializer for Serializer { /// Should non-null default values be specified, a new field could be added to Deserializer #[derive(Clone)] pub struct Deserializer { - required_column_ids: Vec, + required_column_ids: HashMap, schema: Arc<[DataType]>, default_column_values: Vec<(usize, Datum)>, } @@ -189,9 +179,12 @@ impl Deserializer { column_with_default: impl Iterator, ) -> Self { assert_eq!(column_ids.len(), schema.len()); - assert!(column_ids_are_sorted(column_ids)); Self { - required_column_ids: column_ids.iter().map(|id| id.get_id()).collect(), + required_column_ids: column_ids + .iter() + .enumerate() + .map(|(i, c)| (c.get_id(), i)) + .collect::>(), schema, default_column_values: column_with_default.collect(), } @@ -219,15 +212,9 @@ impl ValueRowDeserializer for Deserializer { datums[*i].clone_from(datum); } - // The algorithm here leverages the fact that both `required_column_ids` and the encoded column ids are sorted - // We use two pointers (i and j) to iterate through the two arrays, and only deserialize the columns that are required - let mut j = 0usize; // index of required_column_ids for i in 0..datum_num { let this_id = encoded_bytes.get_i32_le(); - while self.required_column_ids[j] < this_id { - j += 1; // leave the default value as is - } - if self.required_column_ids[j] == this_id { + if let Some(&decoded_idx) = self.required_column_ids.get(&this_id) { let this_offset_start_idx = i * offset_bytes; let mut this_offset_slice = &offsets[this_offset_start_idx..(this_offset_start_idx + offset_bytes)]; @@ -240,15 +227,21 @@ impl ValueRowDeserializer for Deserializer { None } else { let mut data_slice = &data[this_offset..next_offset]; - Some(deserialize_value(&self.schema[j], &mut data_slice)?) + Some(deserialize_value( + &self.schema[decoded_idx], + &mut data_slice, + )?) } } else if this_offset == data.len() { None } else { let mut data_slice = &data[this_offset..]; - Some(deserialize_value(&self.schema[j], &mut data_slice)?) + Some(deserialize_value( + &self.schema[decoded_idx], + &mut data_slice, + )?) }; - datums[j] = data; + datums[decoded_idx] = data; } } Ok(datums) From 3254735bbc0a7cd86625f180ecebd10e1ebc69be Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Fri, 26 Jul 2024 16:23:49 +0800 Subject: [PATCH 6/8] tune performace --- .../src/util/value_encoding/column_aware_row_encoding.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs index 23b6167cdca9..953d4ada764e 100644 --- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs +++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs @@ -19,9 +19,10 @@ //! We have a `Serializer` and a `Deserializer` for each schema of `Row`, which can be reused //! until schema changes -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::sync::Arc; +use ahash::HashMap; use bitflags::bitflags; use super::*; From 8f621641061e25caa88bfaff21aebb881a0a9381 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Tue, 30 Jul 2024 16:07:48 +0800 Subject: [PATCH 7/8] bench 4 columns --- .../bench_column_aware_row_encoding.rs | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/src/common/benches/bench_column_aware_row_encoding.rs b/src/common/benches/bench_column_aware_row_encoding.rs index 91a78ecec7d2..7a6112800c9b 100644 --- a/src/common/benches/bench_column_aware_row_encoding.rs +++ b/src/common/benches/bench_column_aware_row_encoding.rs @@ -22,7 +22,7 @@ use risingwave_common::types::{DataType, Date, ScalarImpl}; use risingwave_common::util::value_encoding::column_aware_row_encoding::*; use risingwave_common::util::value_encoding::*; -fn bench_column_aware_encoding(c: &mut Criterion) { +fn bench_column_aware_encoding_16_columns(c: &mut Criterion) { let mut rng = rand::rngs::StdRng::seed_from_u64(42); // The schema is inspired by the TPC-H lineitem table @@ -67,7 +67,7 @@ fn bench_column_aware_encoding(c: &mut Criterion) { .map(|i| ColumnId::from(i as i32)) .collect::>(); - c.bench_function("bench_column_aware_row_encoding_encode", |b| { + c.bench_function("column_aware_row_encoding_16_columns_encode", |b| { let serializer = Serializer::new(&column_ids[..]); b.iter(|| { black_box(serializer.serialize(&row)); @@ -77,7 +77,7 @@ fn bench_column_aware_encoding(c: &mut Criterion) { let serializer = Serializer::new(&column_ids[..]); let encoded = serializer.serialize(&row); - c.bench_function("bench_column_aware_row_encoding_decode", |b| { + c.bench_function("column_aware_row_encoding_16_columns_decode", |b| { let deserializer = Deserializer::new(&column_ids[..], data_types.clone(), std::iter::empty()); b.iter(|| { @@ -87,5 +87,50 @@ fn bench_column_aware_encoding(c: &mut Criterion) { }); } -criterion_group!(benches, bench_column_aware_encoding); +fn bench_column_aware_encoding_4_columns(c: &mut Criterion) { + let mut rng = rand::rngs::StdRng::seed_from_u64(42); + + // The schema is inspired by the TPC-H nation table + let data_types = Arc::new([ + DataType::Int32, + DataType::Varchar, + DataType::Int32, + DataType::Varchar, + ]); + let row = OwnedRow::new(vec![ + Some(ScalarImpl::Int32(rng.gen())), + Some(ScalarImpl::Utf8("United States".into())), + Some(ScalarImpl::Int32(rng.gen())), + Some(ScalarImpl::Utf8("No comments".into())), + ]); + + let column_ids = (1..=data_types.len()) + .map(|i| ColumnId::from(i as i32)) + .collect::>(); + + c.bench_function("column_aware_row_encoding_4_columns_encode", |b| { + let serializer = Serializer::new(&column_ids[..]); + b.iter(|| { + black_box(serializer.serialize(&row)); + }); + }); + + let serializer = Serializer::new(&column_ids[..]); + let encoded = serializer.serialize(&row); + + c.bench_function("column_aware_row_encoding_4_columns_decode", |b| { + let deserializer = + Deserializer::new(&column_ids[..], data_types.clone(), std::iter::empty()); + b.iter(|| { + let result = deserializer.deserialize(&encoded).unwrap(); + black_box(result); + }); + }); +} + +criterion_group!( + benches, + bench_column_aware_encoding_16_columns, + bench_column_aware_encoding_4_columns, +); criterion_main!(benches); From b76984c7d5491feedfe0ebb652fba12fe43b5ab7 Mon Sep 17 00:00:00 2001 From: Eric Fu Date: Tue, 30 Jul 2024 16:21:23 +0800 Subject: [PATCH 8/8] build a default row in advance --- .../column_aware_row_encoding.rs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/common/src/util/value_encoding/column_aware_row_encoding.rs b/src/common/src/util/value_encoding/column_aware_row_encoding.rs index 953d4ada764e..aea36d4b64da 100644 --- a/src/common/src/util/value_encoding/column_aware_row_encoding.rs +++ b/src/common/src/util/value_encoding/column_aware_row_encoding.rs @@ -170,7 +170,9 @@ impl ValueRowSerializer for Serializer { pub struct Deserializer { required_column_ids: HashMap, schema: Arc<[DataType]>, - default_column_values: Vec<(usize, Datum)>, + + /// A row with default values for each column or `None` if no default value is specified + default_row: Vec, } impl Deserializer { @@ -180,6 +182,10 @@ impl Deserializer { column_with_default: impl Iterator, ) -> Self { assert_eq!(column_ids.len(), schema.len()); + let mut default_row: Vec = vec![None; schema.len()]; + for (i, datum) in column_with_default { + default_row[i] = datum; + } Self { required_column_ids: column_ids .iter() @@ -187,7 +193,7 @@ impl Deserializer { .map(|(i, c)| (c.get_id(), i)) .collect::>(), schema, - default_column_values: column_with_default.collect(), + default_row, } } } @@ -207,12 +213,7 @@ impl ValueRowDeserializer for Deserializer { let offsets = &encoded_bytes[offsets_start_idx..data_start_idx]; let data = &encoded_bytes[data_start_idx..]; - // initialize datums with default values - let mut datums: Vec = vec![None; self.schema.len()]; - for (i, datum) in &self.default_column_values { - datums[*i].clone_from(datum); - } - + let mut row = self.default_row.clone(); for i in 0..datum_num { let this_id = encoded_bytes.get_i32_le(); if let Some(&decoded_idx) = self.required_column_ids.get(&this_id) { @@ -242,10 +243,10 @@ impl ValueRowDeserializer for Deserializer { &mut data_slice, )?) }; - datums[decoded_idx] = data; + row[decoded_idx] = data; } } - Ok(datums) + Ok(row) } }