Skip to content

Commit

Permalink
Fixing NuLazyFrame/NuDataFrame conversion issues (nushell#12538)
Browse files Browse the repository at this point in the history
# Description

@maxim-uvarov brought up another case where converting back and forth
between eager and lazy dataframes was not working correctly:

```
> [[a b]; [6 2] [1 4] [4 1]] | polars into-lazy | polars append -c ([[a b]; [6 2] [1 4] [4 1]] | polars into-df)
Error: nu::shell::cant_convert

  × Can't convert to NuDataFrame.
   ╭─[entry nushell#1:1:49]
 1 │ [[a b]; [6 2] [1 4] [4 1]] | polars into-lazy | polars append -c ([[a b]; [6 2] [1 4] [4 1]] | polars into-df)
   ·                                                 ──────┬──────
   ·                                                       ╰── can't convert NuLazyFrameCustomValue to NuDataFrame
   ╰────
```

This pull request fixes this case and glaringly obvious similar cases I
could find.

Co-authored-by: Jack Wright <[email protected]>
  • Loading branch information
ayax79 and ayax79 authored Apr 16, 2024
1 parent 48e4448 commit a7a5ec3
Show file tree
Hide file tree
Showing 10 changed files with 25 additions and 18 deletions.
2 changes: 1 addition & 1 deletion crates/nu_plugin_polars/src/dataframe/eager/append.rs
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ fn command(
};

let df_other = NuDataFrame::try_from_value_coerce(plugin, &other, call.head)?;
let df = NuDataFrame::try_from_pipeline(plugin, input, call.head)?;
let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;
let df = df.append_df(&df_other, axis, call.head)?;

df.to_pipeline_data(plugin, engine, call.head)
Expand Down
6 changes: 3 additions & 3 deletions crates/nu_plugin_polars/src/dataframe/eager/first.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::{
values::{Column, CustomValueSupport},
values::{Column, CustomValueSupport, NuLazyFrame},
PolarsPlugin,
};

Expand Down Expand Up @@ -98,8 +98,8 @@ impl PluginCommand for FirstDF {
input: PipelineData,
) -> Result<PipelineData, LabeledError> {
let value = input.into_value(call.head);
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(plugin, &value)?;
if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value_coerce(plugin, &value, call.head)?;
command(plugin, engine, call, df).map_err(|e| e.into())
} else {
let expr = NuExpression::try_from_value(plugin, &value)?;
Expand Down
2 changes: 1 addition & 1 deletion crates/nu_plugin_polars/src/dataframe/eager/get.rs
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ fn command(
let columns: Vec<Value> = call.rest(0)?;
let (col_string, col_span) = convert_columns_string(columns, call.head)?;

let df = NuDataFrame::try_from_pipeline(plugin, input, call.head)?;
let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;

let df = df
.as_ref()
Expand Down
6 changes: 3 additions & 3 deletions crates/nu_plugin_polars/src/dataframe/eager/last.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use crate::{
values::{Column, CustomValueSupport},
values::{Column, CustomValueSupport, NuLazyFrame},
PolarsPlugin,
};

Expand Down Expand Up @@ -73,8 +73,8 @@ impl PluginCommand for LastDF {
input: PipelineData,
) -> Result<PipelineData, LabeledError> {
let value = input.into_value(call.head);
if NuDataFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value(plugin, &value)?;
if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) {
let df = NuDataFrame::try_from_value_coerce(plugin, &value, call.head)?;
command(plugin, engine, call, df).map_err(|e| e.into())
} else {
let expr = NuExpression::try_from_value(plugin, &value)?;
Expand Down
2 changes: 1 addition & 1 deletion crates/nu_plugin_polars/src/dataframe/eager/melt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ fn command(
let (id_col_string, id_col_span) = convert_columns_string(id_col, call.head)?;
let (val_col_string, val_col_span) = convert_columns_string(val_col, call.head)?;

let df = NuDataFrame::try_from_pipeline(plugin, input, call.head)?;
let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;

check_column_datatypes(df.as_ref(), &id_col_string, id_col_span)?;
check_column_datatypes(df.as_ref(), &val_col_string, val_col_span)?;
Expand Down
2 changes: 1 addition & 1 deletion crates/nu_plugin_polars/src/dataframe/eager/sample.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,7 @@ fn command(
let replace: bool = call.has_flag("replace")?;
let shuffle: bool = call.has_flag("shuffle")?;

let df = NuDataFrame::try_from_pipeline(plugin, input, call.head)?;
let df = NuDataFrame::try_from_pipeline_coerce(plugin, input, call.head)?;

let df = match (rows, fraction) {
(Some(rows), None) => df
Expand Down
3 changes: 2 additions & 1 deletion crates/nu_plugin_polars/src/dataframe/eager/take.rs
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ fn command(
) -> Result<PipelineData, ShellError> {
let index_value: Value = call.req(0)?;
let index_span = index_value.span();
let index = NuDataFrame::try_from_value(plugin, &index_value)?.as_series(index_span)?;
let index = NuDataFrame::try_from_value_coerce(plugin, &index_value, call.head)?
.as_series(index_span)?;

let casted = match index.dtype() {
DataType::UInt32 | DataType::UInt64 | DataType::Int32 | DataType::Int64 => index
Expand Down
10 changes: 7 additions & 3 deletions crates/nu_plugin_polars/src/dataframe/eager/to_nu.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,11 @@ use nu_protocol::{
SyntaxShape, Type, Value,
};

use crate::{dataframe::values::NuExpression, values::CustomValueSupport, PolarsPlugin};
use crate::{
dataframe::values::NuExpression,
values::{CustomValueSupport, NuLazyFrame},
PolarsPlugin,
};

use super::super::values::NuDataFrame;

Expand Down Expand Up @@ -86,7 +90,7 @@ impl PluginCommand for ToNu {
input: PipelineData,
) -> Result<PipelineData, LabeledError> {
let value = input.into_value(call.head);
if NuDataFrame::can_downcast(&value) {
if NuDataFrame::can_downcast(&value) || NuLazyFrame::can_downcast(&value) {
dataframe_command(plugin, call, value)
} else {
expression_command(plugin, call, value)
Expand All @@ -103,7 +107,7 @@ fn dataframe_command(
let rows: Option<usize> = call.get_flag("rows")?;
let tail: bool = call.has_flag("tail")?;

let df = NuDataFrame::try_from_value(plugin, &input)?;
let df = NuDataFrame::try_from_value_coerce(plugin, &input, call.head)?;

let values = if tail {
df.tail(rows, call.head)?
Expand Down
3 changes: 2 additions & 1 deletion crates/nu_plugin_polars/src/dataframe/eager/with_column.rs
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ fn command_eager(
let df = lazy.collect(call.head)?;
df.to_pipeline_data(plugin, engine, call.head)
} else {
let mut other = NuDataFrame::try_from_value(plugin, &new_column)?.as_series(column_span)?;
let mut other = NuDataFrame::try_from_value_coerce(plugin, &new_column, call.head)?
.as_series(column_span)?;

let name = match call.get_flag::<String>("name")? {
Some(name) => name,
Expand Down
7 changes: 4 additions & 3 deletions crates/nu_plugin_polars/src/dataframe/expressions/is_in.rs
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ fn command_df(
) -> Result<PipelineData, ShellError> {
let other_value: Value = call.req(0)?;
let other_span = other_value.span();
let other_df = NuDataFrame::try_from_value(plugin, &other_value)?;
let other_df = NuDataFrame::try_from_value_coerce(plugin, &other_value, call.head)?;
let other = other_df.as_series(other_span)?;
let series = df.as_series(call.head)?;

Expand All @@ -181,8 +181,9 @@ fn command_df(

res.rename("is_in");

let df = NuDataFrame::try_from_series_vec(vec![res.into_series()], call.head)?;
df.to_pipeline_data(plugin, engine, call.head)
let mut new_df = NuDataFrame::try_from_series_vec(vec![res.into_series()], call.head)?;
new_df.from_lazy = df.from_lazy;
new_df.to_pipeline_data(plugin, engine, call.head)
}

#[cfg(test)]
Expand Down

0 comments on commit a7a5ec3

Please sign in to comment.