Skip to content

Commit

Permalink
ShowRelation to print out the relation expression (#45)
Browse files Browse the repository at this point in the history
* ShowRelation to print out the relation expression

* docs
  • Loading branch information
GoEddie authored Dec 2, 2024
1 parent dcc57bb commit cfc248b
Show file tree
Hide file tree
Showing 2 changed files with 30 additions and 2 deletions.
25 changes: 25 additions & 0 deletions src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/DataFrame.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using StructType = Spark.Connect.Dotnet.Sql.Types.StructType;
using static Spark.Connect.Dotnet.Sql.Functions;
using TransformFunction = System.Linq.Expressions.Expression<System.Func<Spark.Connect.Dotnet.Sql.DataFrame, Spark.Connect.Dotnet.Sql.DataFrame>>;
using Newtonsoft.Json;

namespace Spark.Connect.Dotnet.Sql;

Expand Down Expand Up @@ -1718,6 +1719,30 @@ public DataFrame Transform(TransformFunction function)

return response;
}

/// <summary>
/// Prints out the Relation representation, if you don't want it printed to console then you can get it by doing `DataFrame.Relation.ToString()`
/// </summary>
public void ShowRelation()
{
try
{
using (var stringReader = new StringReader(this.Relation.ToString()))
using (var stringWriter = new StringWriter())
{
var jsonReader = new JsonTextReader(stringReader);
var jsonWriter = new JsonTextWriter(stringWriter) { Formatting = Formatting.Indented };
jsonWriter.WriteToken(jsonReader);
SparkSession.Console.WriteLine(stringWriter.ToString());
}
}
catch (Exception)
{
var json = this.Relation.ToString();
SparkSession.Console.WriteLine(this.Relation.ToString());
}

}
}

public enum JoinType
Expand Down
7 changes: 5 additions & 2 deletions src/example/basic_example/SimpleExample.cs
Original file line number Diff line number Diff line change
Expand Up @@ -47,12 +47,15 @@ public async Task Run()

var parquetPath = Path.Join(tempOutputFolder, "parquetFile");

dataFrame2
.WithColumn("id multiplied by 1000", Functions.Col("id") * 1000)
var df3 = dataFrame2.WithColumn("id multiplied by 1000", Functions.Col("id") * 1000);

df3
.Write()
.Format("parquet")
.Write(parquetPath);

Console.WriteLine($"Wrote Parquet to '{parquetPath}'");

df3.ShowRelation();
}
}

0 comments on commit cfc248b

Please sign in to comment.