diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs index 4ce2040..4e224ea 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs @@ -1,7 +1,15 @@ namespace Spark.Connect.Dotnet.Databricks; +/// +/// Are we connecting to Databricks? used to control whether we wait for a connection. +/// public class IsDatabricks { + /// + /// Valiudates whether the url contains the name 'databricks' + /// + /// + /// public static bool Url(string url) { return url.Contains("databricks", StringComparison.OrdinalIgnoreCase); diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs index 68764f5..964f700 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs @@ -7,8 +7,23 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// This is the class used to pass messages down the gRPC channel. +/// public static class GrpcInternal { + /// + /// Explain + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// public static string Explain(SparkConnectService.SparkConnectServiceClient client, string sessionId, Plan plan, Metadata headers, UserContext userContext, string clientType, bool explainExtended, string? mode) { @@ -34,6 +49,13 @@ public static string Explain(SparkConnectService.SparkConnectServiceClient clien return analyzeResponse.Explain.ExplainString; } + /// + /// Persist + /// + /// + /// + /// + /// public static Relation Persist(SparkSession session, Relation relation, StorageLevel storageLevel) { var analyzeRequest = new AnalyzePlanRequest @@ -49,6 +71,18 @@ public static Relation Persist(SparkSession session, Relation relation, StorageL return relation; } + /// + /// Schema + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// public static DataType Schema(SparkConnectService.SparkConnectServiceClient client, string sessionId, Plan plan, Metadata headers, UserContext userContext, string clientType, bool explainExtended, string mode) { @@ -65,6 +99,11 @@ public static DataType Schema(SparkConnectService.SparkConnectServiceClient clie return analyzeResponse.Schema.Schema_; } + /// + /// What is the Spark Version you are connect to? + /// + /// + /// public static string Version(SparkSession session) { var analyzeRequest = new AnalyzePlanRequest @@ -76,6 +115,12 @@ public static string Version(SparkSession session) return analyzeResponse.SparkVersion.Version; } + /// + /// Get a list of the input files + /// + /// + /// + /// public static IEnumerable InputFiles(SparkSession session, Plan plan) { var analyzeRequest = new AnalyzePlanRequest @@ -91,6 +136,12 @@ public static IEnumerable InputFiles(SparkSession session, Plan plan) return analyzeResponse.InputFiles.Files.Select(p => p); } + /// + /// Is it a local plan? + /// + /// + /// + /// public static bool IsLocal(SparkSession session, Plan plan) { var analyzeRequest = new AnalyzePlanRequest @@ -106,6 +157,13 @@ public static bool IsLocal(SparkSession session, Plan plan) return analyzeResponse.IsLocal.IsLocal_; } + /// + /// Get the TreeString + /// + /// + /// + /// + /// public static string TreeString(SparkSession session, Relation relation, int? level = null) { var analyzeRequest = new AnalyzePlanRequest @@ -129,6 +187,12 @@ public static string TreeString(SparkSession session, Relation relation, int? le return analyzeResponse.TreeString.TreeString_; } + /// + /// Create a semantic hash of the relation + /// + /// + /// + /// public static int SemanticHash(SparkSession session, Relation relation) { var analyzeRequest = new AnalyzePlanRequest @@ -148,6 +212,12 @@ public static int SemanticHash(SparkSession session, Relation relation) return analyzeResponse.SemanticHash.Result; } + /// + /// What is the storage level? + /// + /// + /// + /// public static StorageLevel StorageLevel(SparkSession session, Relation relation) { var analyzeRequest = new AnalyzePlanRequest @@ -164,6 +234,12 @@ public static StorageLevel StorageLevel(SparkSession session, Relation relation) return analyzeResponse.GetStorageLevel.StorageLevel; } + /// + /// Is it a Streaming plan + /// + /// + /// + /// public static bool IsStreaming(SparkSession session, Plan plan) { var analyzeRequest = new AnalyzePlanRequest @@ -179,6 +255,13 @@ public static bool IsStreaming(SparkSession session, Plan plan) return analyzeResponse.IsStreaming.IsStreaming_; } + /// + /// Same Semantics, uses AnalyzePlanRequest + /// + /// + /// + /// + /// public static bool SameSemantics(SparkSession session, Relation target, Relation other) { var analyzeRequest = new AnalyzePlanRequest @@ -201,6 +284,12 @@ public static bool SameSemantics(SparkSession session, Relation target, Relation return analyzeResponse.SameSemantics.Result; } + /// + /// Unset a config option + /// + /// + /// + /// public static async Task ExecUnSetConfigCommandResponse(SparkSession session, string key) { var configRequest = new ConfigRequest @@ -245,6 +334,12 @@ AsyncUnaryCall Exec() } } + /// + /// Set Config Item + /// + /// + /// + /// public static async Task ExecSetConfigCommandResponse(SparkSession session, IDictionary options) { var configRequest = new ConfigRequest @@ -295,6 +390,13 @@ AsyncUnaryCall Exec() } } + /// + /// Get All Config Response + /// + /// + /// + /// + /// public static async Task> ExecGetAllConfigCommandResponse(SparkSession session, string? prefix = null) { diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs index 7250a37..a0abb03 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs @@ -1,10 +1,18 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Used to log gRPC info, needs migrating to a better logger. +/// public class GrpcLogger { private readonly GrpcLoggingLevel _level; private readonly LocalConsole _console; + /// + /// Create a logger, pass in your own console if you want to redirect output (like in tests!) + /// + /// + /// public GrpcLogger(GrpcLoggingLevel level, LocalConsole? console = null) { _level = level; @@ -13,6 +21,11 @@ public GrpcLogger(GrpcLoggingLevel level, LocalConsole? console = null) } + /// + /// Log + /// + /// + /// public virtual void Log(GrpcLoggingLevel level, string message) { if (level >= _level) @@ -21,6 +34,12 @@ public virtual void Log(GrpcLoggingLevel level, string message) } } + /// + /// Log + /// + /// + /// + /// public virtual void Log(GrpcLoggingLevel level, string format, params object[] args) { if (level >= _level) @@ -30,26 +49,57 @@ public virtual void Log(GrpcLoggingLevel level, string format, params object[] a } } +/// +/// The default logger that does nothing +/// public class GrpcNullLogger : GrpcLogger { + /// + /// Create the default null logger + /// + /// + /// public GrpcNullLogger(GrpcLoggingLevel level, LocalConsole? console = null) : base(level, console) { } + /// + /// Log + /// + /// + /// + /// public override void Log(GrpcLoggingLevel level, string format, params object[] args) { } + /// + /// Log + /// + /// + /// public override void Log(GrpcLoggingLevel level, string message) { } } +/// +/// Log Level +/// public enum GrpcLoggingLevel { + /// + /// None + /// None, + /// + /// Warn + /// Warn, + /// + /// Verbose + /// Verbose } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs index 74b25a7..2b17354 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs @@ -1,14 +1,25 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Logger +/// public class Logger { private static int _level; + /// + /// Create the logger at whatever level you like + /// + /// public Logger(int level) { _level = level; } + /// + /// Log + /// + /// public static void WriteLine(string message) { if (_level > 1) @@ -17,6 +28,11 @@ public static void WriteLine(string message) } } + /// + /// Log + /// + /// + /// public static void WriteLine(string message, params object[] parameters) { if (_level > 1) diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs index ff7384c..689e371 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs @@ -4,21 +4,24 @@ namespace Spark.Connect.Dotnet.Grpc; -/* - * When we connect to remote spark clusters, including Databricks we can find our tcp connections are closed - * Azure has a 5 minute idle time where connections are killed and Databricks has a hard 1 hour timeout so we - * need to create the initial request with the Reattach option Reattachable set to true and then we can re-connect - * a failed connection using ReattachExecute so we don't have to re-run the query from the beginning. - * - * The .NET gRPC request will sit forever and not respond if the connection is killed so we use a scheduled cancellation - * to cancel the request if we haven't had a response in 1 minute. - * - * When we get the first response it includes an operation id and a request id - we use the operation id to identify the query run - * and the response id to track the responses we have received. the response id is like a pointer to tell the server where we have - * received the response up to. - * - * When we finish and have everything we need we should tell the server to release the responses so the memory is freed on the server. - */ + + + +/// +/// When we connect to remote spark clusters, including Databricks we can find our tcp connections are closed +/// Azure has a 5 minute idle time where connections are killed and Databricks has a hard 1 hour timeout so we +/// need to create the initial request with the Reattach option Reattachable set to true and then we can re-connect +/// a failed connection using ReattachExecute so we don't have to re-run the query from the beginning. +/// +/// The .NET gRPC request will sit forever and not respond if the connection is killed so we use a scheduled cancellation +/// to cancel the request if we haven't had a response in 1 minute. +/// +/// When we get the first response it includes an operation id and a request id - we use the operation id to identify the query run +/// and the response id to track the responses we have received. the response id is like a pointer to tell the server where we have +/// received the response up to. +/// +/// When we finish and have everything we need we should tell the server to release the responses so the memory is freed on the server. +/// public class RequestExecutor : IDisposable { private readonly SparkSession _session; @@ -49,6 +52,11 @@ private enum RetryableState private RetryableState _retryableState = RetryableState.Processing; + /// + /// Create the Executor + /// + /// + /// public RequestExecutor(SparkSession session, Plan plan) { _logger = GetLogger(session); @@ -71,12 +79,18 @@ private GrpcLogger GetLogger(SparkSession session) return new GrpcNullLogger(GrpcLoggingLevel.None, null); } + /// + /// Excecute the plan passed in the constructor + /// public void Exec() { var task = Task.Run(ExecAsync); task.Wait(); } + /// + /// Execute the plan already passed into the constructor + /// public async Task ExecAsync() { var shouldContinue = true; @@ -353,6 +367,9 @@ private void PrintObservedMetrics(RepeatedField + /// Dispose releasing any outstanding requests + /// public void Dispose() { if (_operationId != string.Empty && _lastResponseId != String.Empty) @@ -367,22 +384,58 @@ public void Dispose() } } + /// + /// Get any data returned by the last request + /// + /// public IList GetData() => _rows; + /// + /// The schema of the last request (if available) + /// + /// public DataType GetSchema() => _schema!; + /// + /// If there was a relation returned + /// + /// public Relation GetRelation() => _relation!; + /// + /// Get the streaming query id + /// + /// public StreamingQueryInstanceId GetStreamingQueryId() => _streamingQueryId!; + /// + /// Streaming command result + /// + /// public StreamingQueryCommandResult.Types.StatusResult GetStreamingQueryCommandResult() => _streamingResultStatus!; + /// + /// Streaming query name + /// + /// public string GetStreamingQueryName() => _streamingQueryName!; + /// + /// Is the streaming query terminated? + /// + /// public bool GetStreamingQueryIsTerminated() => _streamingQueryIsTerminated!.Value; + /// + /// Any streaming exceptions + /// + /// public StreamingQueryCommandResult.Types.ExceptionResult? GetStreamingException() => _streamingQueryException; + /// + /// Any streaming progress info + /// + /// public StreamingQueryCommandResult.Types.RecentProgressResult? GetStreamingRecentProgress() => _streamingProgress; } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs index 5708718..3e6f86e 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs @@ -1,13 +1,25 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Wrap DATATYPE_MISMATCH.NON_FOLDABLE_INPUT +/// public class DataTypeMismatchNonFoldableInputException : SparkException { //DATATYPE_MISMATCH.NON_FOLDABLE_INPUT + /// + /// Wrap DATATYPE_MISMATCH.NON_FOLDABLE_INPUT + /// + /// + /// public DataTypeMismatchNonFoldableInputException(string exceptionMessage, Exception exception) : base( exceptionMessage, exception) { } + /// + /// Wrap DATATYPE_MISMATCH.NON_FOLDABLE_INPUT + /// + /// public DataTypeMismatchNonFoldableInputException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs index 5fa43ba..681be41 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs @@ -1,7 +1,15 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Wrap [DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] +/// public class DataTypeMismatchUnexpectedInputTypeException : SparkException { + /// + /// Wrap [DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] + /// + /// + /// public DataTypeMismatchUnexpectedInputTypeException(string exceptionMessage, Exception exception) : base( exceptionMessage, exception) { @@ -9,6 +17,10 @@ public DataTypeMismatchUnexpectedInputTypeException(string exceptionMessage, Exc OverrideMessage = exceptionMessage.Replace("[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE]", ""); } + /// + /// Wrap [DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] + /// + /// public DataTypeMismatchUnexpectedInputTypeException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs index 893412d..cb408b4 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs @@ -1,12 +1,25 @@ namespace Spark.Connect.Dotnet.Grpc; + +/// +/// Wrap GROUP_BY_AGGREGATE +/// public class GroupByAggregateException : SparkException { //GROUP_BY_AGGREGATE + /// + /// Wrap GROUP_BY_AGGREGATE + /// + /// + /// public GroupByAggregateException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } + /// + /// Wrap GROUP_BY_AGGREGATE + /// + /// public GroupByAggregateException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs index 58fafd3..ac6aacb 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs @@ -1,12 +1,24 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Generic internal error +/// public class InternalSparkException : SparkException { //Internal + /// + /// Generic internal error + /// + /// + /// public InternalSparkException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } + /// + /// Generic internal error + /// + /// public InternalSparkException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs index f8f16b1..70f76ec 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs @@ -1,11 +1,23 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Wrap MISSING_GROUP_BY +/// public class MissingGroupByException : SparkException { + /// + /// Wrap MISSING_GROUP_BY + /// + /// + /// public MissingGroupByException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } + /// + /// Wrap MISSING_GROUP_BY + /// + /// public MissingGroupByException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs index 7f1ac4f..681f352 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs @@ -2,10 +2,22 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Helper exceptions for any errors we get +/// public class SparkException : Exception { + /// + /// Implementing exceptions can override the default error message + /// protected string OverrideMessage = ""; + + /// + /// Helper exceptions for any errors we get + /// + /// + /// public SparkException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { if (exception is AggregateException && exception.InnerException is RpcException rpcException) @@ -19,6 +31,10 @@ public SparkException(string exceptionMessage, Exception exception) : base(excep } } + /// + /// Helper exceptions for any errors we get + /// + /// public SparkException(Exception exception) : base(exception.Message, exception) { if (exception is AggregateException && exception.InnerException is RpcException rpcException) @@ -27,10 +43,17 @@ public SparkException(Exception exception) : base(exception.Message, exception) } } + /// + /// Helper exceptions for any errors we get + /// + /// public SparkException(string message) : base(message) { OverrideMessage = message; } + /// + /// Error message + /// public override string Message => OverrideMessage; } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs index 4c5eac9..c0256bf 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs @@ -2,6 +2,9 @@ namespace Spark.Connect.Dotnet.Grpc.SparkExceptions; +/// +/// Used to help map and create exceptions +/// public static class SparkExceptionFactory { private static SparkException DetailStringToException(string exceptionCode, string detail, Exception exception) @@ -32,6 +35,11 @@ private static string ExceptionCodeFromSparkError(string detail, string statusCo return exceptionCode; } + /// + /// Create the appropriate exception + /// + /// + /// public static SparkException GetExceptionFromRpcException(RpcException exception) { var exceptionCode = @@ -39,6 +47,11 @@ public static SparkException GetExceptionFromRpcException(RpcException exception return DetailStringToException(exceptionCode, exception.Status.Detail, exception); } + /// + /// Create the appropriate exception + /// + /// + /// public static SparkException GetExceptionFromRpcException(AggregateException aggException) { if (aggException.InnerException is RpcException exception) diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs new file mode 100644 index 0000000..689c186 --- /dev/null +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs @@ -0,0 +1,22 @@ +namespace Spark.Connect.Dotnet.Grpc; + +/// +/// Wrap Unavailable +/// +public class UnavailableException : SparkException +{ + //"Unavailable" + + /// + /// Wrap Unavailable + /// + /// + /// + public UnavailableException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) + { + if (exceptionMessage == "Error connecting to subchannel.") + { + OverrideMessage = "Cannot connect to remote Spark Server, if your $SPARK_REMOTE set correctly?"; + } + } +} \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs index d6921a7..f8fcb0e 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs @@ -1,12 +1,19 @@ using System.Text.RegularExpressions; -using Spark.Connect.Dotnet.Sql; namespace Spark.Connect.Dotnet.Grpc; +/// +/// Wrap UNRESOLVED_COLUMN.WITH_SUGGESTION +/// public class UnresolvedColumnWithSuggestionException : SparkException { //UNRESOLVED_COLUMN.WITH_SUGGESTION + /// + /// Wrap UNRESOLVED_COLUMN.WITH_SUGGESTION + /// + /// + /// public UnresolvedColumnWithSuggestionException(string exceptionMessage, Exception exception) : base( exceptionMessage, exception) { @@ -25,29 +32,11 @@ public UnresolvedColumnWithSuggestionException(string exceptionMessage, Exceptio OverrideMessage = $@"The Column `{columnName}` was not found, did you mean one of: {existingColumns}?"; } + /// + /// Wrap UNRESOLVED_COLUMN.WITH_SUGGESTION + /// + /// public UnresolvedColumnWithSuggestionException(Exception exception) : base(exception) { } -} - -public class WindowFunctionWithoutOverException : SparkException -{ - //WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE - - public WindowFunctionWithoutOverException(string exceptionMessage, Exception exception) : base(exceptionMessage, - exception) - { - //Format = [WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE] Window function "(.*?)" requires an OVER clause. - var input = exceptionMessage; - // Extract the column name - var windowFunctionPattern = @"Window function ""(.*?)"""; - var windowFunction = Regex.Match(input, windowFunctionPattern).Value; - - OverrideMessage = - $@"The Window Function `{FunctionsWrapper.CSharpFunctionName(windowFunction)}` requires an OVER clause"; - } - - public WindowFunctionWithoutOverException(Exception exception) : base(exception) - { - } } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs index 6585c53..bf281ba 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs @@ -1,23 +1,18 @@ namespace Spark.Connect.Dotnet.Grpc; +/// +/// Wrap UNRESOLVED_ROUTINE +/// public class UnresolvedRoutineException : SparkException { //"UNRESOLVED_ROUTINE" + /// + /// Wrap UNRESOLVED_ROUTINE + /// + /// + /// public UnresolvedRoutineException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } -} - -public class UnavailableException : SparkException -{ - //"Unavailable" - - public UnavailableException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) - { - if (exceptionMessage == "Error connecting to subchannel.") - { - OverrideMessage = "Cannot connect to remote Spark Server, if your $SPARK_REMOTE set correctly?"; - } - } } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs new file mode 100644 index 0000000..d5051ca --- /dev/null +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs @@ -0,0 +1,38 @@ +using System.Text.RegularExpressions; +using Spark.Connect.Dotnet.Sql; + +namespace Spark.Connect.Dotnet.Grpc; + +/// +/// Wrap WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE +/// +public class WindowFunctionWithoutOverException : SparkException +{ + //WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE + + /// + /// Wrap WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE + /// + /// + /// + public WindowFunctionWithoutOverException(string exceptionMessage, Exception exception) : base(exceptionMessage, + exception) + { + //Format = [WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE] Window function "(.*?)" requires an OVER clause. + var input = exceptionMessage; + // Extract the column name + var windowFunctionPattern = @"Window function ""(.*?)"""; + var windowFunction = Regex.Match(input, windowFunctionPattern).Value; + + OverrideMessage = + $@"The Window Function `{FunctionsWrapper.CSharpFunctionName(windowFunction)}` requires an OVER clause"; + } + + /// + /// Wrap WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE + /// + /// + public WindowFunctionWithoutOverException(Exception exception) : base(exception) + { + } +} \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs index c71b4f1..559da63 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs @@ -1,12 +1,23 @@ namespace Spark.Connect.Dotnet; +/// +/// This is used by our tests to redirect stdout to the console +/// public class LocalConsole { + /// + /// Write to the console + /// + /// public virtual void Write(string what) { Console.Write(what); } + /// + /// WriteLine to the console + /// + /// public virtual void WriteLine(string what) { Console.WriteLine(what); diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs index 36ada7b..997f6ff 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs @@ -2,21 +2,44 @@ namespace Spark.Connect.Dotnet.Sql; +/// +/// A column in a DataFrame. +/// +/// This wraps an `Expression` which is what we pass to Spark Connect. +/// public class Column { private readonly string _name = string.Empty; + + /// + /// This is what the Column actually does, it is what is passed to spark + /// public readonly Expression Expression; + /// + /// Get a column by name. Static version used by the functions and wraps `Column(name)` + /// + /// + /// Column public static Column Col(string name) { return new Column(name); } + /// + /// Create a `Column` from an `Expression` + /// + /// public Column(Expression expression) { Expression = expression; } + /// + /// Pass either the name of a column, or *. You can also pass a `DataFrame` to use as the source. + /// + /// + /// public Column(string name, DataFrame? source = null) { _name = name; @@ -72,6 +95,12 @@ private bool Equals(Column other) return Expression.Equals(other.Expression); } + + /// + /// Does this column equal the passed in object? + /// + /// + /// public override bool Equals(object? obj) { if (ReferenceEquals(null, obj)) @@ -87,11 +116,20 @@ public override bool Equals(object? obj) return obj.GetType() == GetType() && Equals((Column)obj); } + /// + /// Get the hash code of the column (hopefully no one needs this, if they do then it sounds like a debugging nightmare) + /// + /// public override int GetHashCode() { return Expression.GetHashCode(); } + /// + /// Alias the column to another name + /// + /// + /// Column public Column Alias(string name) { var expression = new Expression @@ -105,11 +143,20 @@ public Column Alias(string name) return new Column(expression); } + /// + /// Alias the column to another name + /// + /// The new name + /// Column public Column As(string name) { return Alias(name); } + /// + /// Order the existing column ascending + /// + /// public Column Asc() { return new Column(new Expression @@ -121,6 +168,10 @@ public Column Asc() }); } + /// + /// Order the existing column descending + /// + /// public Column Desc() { return new Column(new Expression @@ -132,909 +183,1880 @@ public Column Desc() }); } + /// + /// & the column with the passed in bool + /// + /// + /// + /// public static Column operator &(Column src, bool value) { return src.And(value); } + /// + /// & the column with the passed in `Column` + /// + /// + /// + /// public static Column operator &(Column src, Column value) { return src.And(value); } + /// + /// & the column with the passed in bool + /// + /// + /// public Column And(bool value) { return BinaryOperation(value, "and"); } + /// + /// & the column with the passed in `Column` + /// + /// + /// public Column And(Column value) { return BinaryOperation(value, "and"); } + /// + /// | the column with the passed in bool + /// + /// + /// + /// public static Column operator |(Column src, bool value) { return src.Or(value); } + /// + /// | the column with the passed in `Column` + /// + /// + /// + /// public static Column operator |(Column src, Column value) { return src.Or(value); } + /// + /// | the column with the passed in bool + /// + /// + /// public Column Or(bool value) { return BinaryOperation(value, "or"); } + /// + /// | the column with the passed in `Column` + /// + /// + /// public Column Or(Column value) { return BinaryOperation(value, "or"); } + /// + /// ! the column + /// + /// + /// public static Column operator !(Column src) { return src.Not(); } + /// + /// ! the column + /// + /// public Column Not() { return BinaryOperation("not"); } + /// + /// Multiply the column with the value + /// + /// + /// + /// public static Column operator *(Column src, int value) { return src.Multiply(value); } + /// + /// Multiply the column with the value + /// + /// + /// + /// public static Column operator *(Column src, double right) { return src.Multiply(right); } + /// + /// Multiply the column with the value + /// + /// + /// + /// public static Column operator *(Column src, float right) { return src.Multiply(right); } + /// + /// Multiply the column with the value + /// + /// + /// + /// public static Column operator *(Column src, long right) { return src.Multiply(right); } + /// + /// Multiply the column with the value + /// + /// + /// + /// public static Column operator *(Column src, short right) { return src.Multiply(right); } + /// + /// Multiply the column with the other column + /// + /// + /// + /// public static Column operator *(Column src, Column value) { return src.Multiply(value); } + /// + /// Multiply the column with the value + /// + /// + /// public Column Multiply(int value) { return BinaryOperation(value, "*"); } + /// + /// Multiply the column with the value + /// + /// + /// public Column Multiply(float value) { return BinaryOperation(value, "*"); } + /// + /// Multiply the column with the value + /// + /// + /// public Column Multiply(double value) { return BinaryOperation(value, "*"); } + /// + /// Multiply the column with the value + /// + /// + /// public Column Multiply(long value) { return BinaryOperation(value, "*"); } + /// + /// Multiply the column with the value + /// + /// + /// public Column Multiply(short value) { return BinaryOperation(value, "*"); } + /// + /// Multiply the column with the value + /// + /// + /// public Column Multiply(Column value) { return BinaryOperation(value, "*"); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, int value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, float value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, double value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, bool value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, long value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, string value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// + /// public static Column operator ==(Column src, Column value) { return src.EqualTo(value); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(string value) { return BinaryOperation(value, "=="); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(int value) { return BinaryOperation(value, "=="); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(float value) { return BinaryOperation(value, "=="); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(double value) { return BinaryOperation(value, "=="); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(bool value) { return BinaryOperation(value, "=="); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(long value) { return BinaryOperation(value, "=="); } + /// + /// Does the column equal the other one? + /// + /// + /// public Column EqualTo(Column value) { return BinaryOperation(value, "=="); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, int value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, string value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, long value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, float value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, double value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, bool value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// + /// public static Column operator !=(Column src, Column value) { return src.NotEqualTo(value); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(int value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(string value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(float value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(double value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(bool value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(long value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// Does the column not equal the other one? + /// + /// + /// public Column NotEqualTo(Column value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// + /// RMultiply by + /// + /// + /// public Column RMultiply(int value) { return BinaryOperation(value, "*", true); } + /// + /// RMultiply by + /// + /// + /// public Column RMultiply(Column value) { return BinaryOperation(value, "*", true); } - + + /// + /// Divide by + /// + /// + /// + /// public static Column operator /(Column src, int value) { return src.Divide(value); } + /// + /// Divide by + /// + /// + /// + /// public static Column operator /(Column src, float value) { return src.Divide(value); } + /// + /// Divide by + /// + /// + /// + /// public static Column operator /(Column src, double value) { return src.Divide(value); } + /// + /// Divide by + /// + /// + /// + /// public static Column operator /(Column src, short value) { return src.Divide(value); } + /// + /// Divide by + /// + /// + /// + /// public static Column operator /(Column src, long value) { return src.Divide(value); } - public static Column operator /(Column src, Column value) - { - return src.Divide(value); + /// + /// Divide by + /// + /// + /// + /// + public static Column operator /(Column src, Column value) + { + return src.Divide(value); } + /// + /// Divide by + /// + /// + /// public Column Divide(int value) { return BinaryOperation(value, "/"); } + /// + /// Divide by + /// + /// + /// public Column Divide(float value) { return BinaryOperation(value, "/"); } + /// + /// Divide by + /// + /// + /// public Column Divide(double value) { return BinaryOperation(value, "/"); } + /// + /// Divide by + /// + /// + /// public Column Divide(short value) { return BinaryOperation(value, "/"); } + /// + /// Divide by + /// + /// + /// public Column Divide(long value) { return BinaryOperation(value, "/"); } + /// + /// Divide by + /// + /// + /// public Column Divide(Column value) { return BinaryOperation(value, "/"); } + /// + /// RDivide by + /// + /// + /// public Column RDivide(int value) { return BinaryOperation(value, "/", true); } + /// + /// RDivide by + /// + /// + /// public Column RDivide(Column value) { return BinaryOperation(value, "/", true); } + /// + /// Add + /// + /// + /// + /// public static Column operator +(Column src, int value) { return src.Add(value); } + /// + /// Add + /// + /// + /// + /// public static Column operator +(Column src, float value) { return src.Add(value); } + /// + /// Add + /// + /// + /// + /// public static Column operator +(Column src, double value) { return src.Add(value); } + /// + /// Add + /// + /// + /// + /// public static Column operator +(Column src, short value) { return src.Add(value); } + /// + /// Add + /// + /// + /// + /// public static Column operator +(Column src, long value) { return src.Add(value); } + /// + /// Add + /// + /// + /// + /// public static Column operator +(Column src, Column value) { return src.Add(value); } + /// + /// Add + /// + /// + /// public Column Add(int value) { return BinaryOperation(value, "+"); } + /// + /// Add + /// + /// + /// public Column Add(float value) { return BinaryOperation(value, "+"); } + /// + /// Add + /// + /// + /// public Column Add(double value) { return BinaryOperation(value, "+"); } + /// + /// Add + /// + /// + /// public Column Add(short value) { return BinaryOperation(value, "+"); } + /// + /// Add + /// + /// + /// public Column Add(long value) { return BinaryOperation(value, "+"); } + /// + /// Add + /// + /// + /// public Column Add(Column value) { return BinaryOperation(value, "+"); } - + + /// + /// RAdd + /// + /// + /// public Column RAdd(int value) { return BinaryOperation(value, "+", true); } + /// + /// RAdd + /// + /// + /// public Column RAdd(float value) { return BinaryOperation(value, "+", true); } + /// + /// RAdd + /// + /// + /// public Column RAdd(double value) { return BinaryOperation(value, "+", true); } + /// + /// RAdd + /// + /// + /// public Column RAdd(short value) { return BinaryOperation(value, "+", true); } + /// + /// RAdd + /// + /// + /// public Column RAdd(long value) { return BinaryOperation(value, "+", true); } + /// + /// RAdd + /// + /// + /// public Column RAdd(Column value) { return BinaryOperation(value, "+", true); } + /// + /// Subtract + /// + /// + /// + /// public static Column operator -(Column src, int value) { return src.Minus(value); } + /// + /// Subtract + /// + /// + /// + /// public static Column operator -(Column src, float value) { return src.Minus(value); } + /// + /// Subtract + /// + /// + /// + /// public static Column operator -(Column src, double value) { return src.Minus(value); } + /// + /// Subtract + /// + /// + /// + /// public static Column operator -(Column src, short value) { return src.Minus(value); } + /// + /// Subtract + /// + /// + /// + /// public static Column operator -(Column src, long value) { return src.Minus(value); } + /// + /// Subtract + /// + /// + /// + /// public static Column operator -(Column src, Column value) { return src.Minus(value); } + /// + /// Subtract + /// + /// + /// public Column Minus(int value) { return BinaryOperation(value, "-"); } + /// + /// Subtract + /// + /// + /// public Column Minus(float value) { return BinaryOperation(value, "-"); } + /// + /// Subtract + /// + /// + /// public Column Minus(double value) { return BinaryOperation(value, "-"); } + /// + /// Subtract + /// + /// + /// public Column Minus(short value) { return BinaryOperation(value, "-"); } + /// + /// Subtract + /// + /// + /// public Column Minus(long value) { return BinaryOperation(value, "-"); } + /// + /// Subtract + /// + /// + /// public Column Minus(Column value) { return BinaryOperation(value, "-"); } + /// + /// R Subtract + /// + /// + /// public Column RMinus(int value) { return BinaryOperation(value, "-", true); } + /// + /// R Subtract + /// + /// + /// public Column RMinus(float value) { return BinaryOperation(value, "-", true); } - + + /// + /// R Subtract + /// + /// + /// public Column RMinus(double value) { return BinaryOperation(value, "-", true); } + /// + /// R Subtract + /// + /// + /// public Column RMinus(short value) { return BinaryOperation(value, "-", true); } + /// + /// R Subtract + /// + /// + /// public Column RMinus(long value) { return BinaryOperation(value, "-", true); } + /// + /// R Subtract + /// + /// + /// public Column RMinus(Column value) { return BinaryOperation(value, "-", true); } - + + /// + /// Mod + /// + /// + /// + /// public static Column operator %(Column src, int value) { return src.Mod(value); } + /// + /// Mod + /// + /// + /// + /// public static Column operator %(Column src, float value) { return src.Mod(value); } + /// + /// Mod + /// + /// + /// + /// public static Column operator %(Column src, double value) { return src.Mod(value); } + /// + /// Mod + /// + /// + /// + /// public static Column operator %(Column src, short value) { return src.Mod(value); } + /// + /// Mod + /// + /// + /// + /// public static Column operator %(Column src, long value) { return src.Mod(value); } + /// + /// Mod + /// + /// + /// + /// public static Column operator %(Column src, Column value) { return src.Mod(value); } + /// + /// Mod + /// + /// + /// public Column Mod(int value) { return BinaryOperation(value, "%"); } + /// + /// Mod + /// + /// + /// public Column Mod(float value) { return BinaryOperation(value, "%"); } + /// + /// Mod + /// + /// + /// public Column Mod(double value) { return BinaryOperation(value, "%"); } + /// + /// Mod + /// + /// + /// public Column Mod(short value) { return BinaryOperation(value, "%"); } + /// + /// Mod + /// + /// + /// public Column Mod(long value) { return BinaryOperation(value, "%"); } + /// + /// Mod + /// + /// + /// public Column Mod(Column value) { return BinaryOperation(value, "%"); } + /// + /// R Mod + /// + /// + /// public Column RMod(int value) { return BinaryOperation(value, "%", true); } + /// + /// R Mod + /// + /// + /// public Column RMod(float value) { return BinaryOperation(value, "%", true); } + /// + /// R Mod + /// + /// + /// public Column RMod(double value) { return BinaryOperation(value, "%", true); } + /// + /// R Mod + /// + /// + /// public Column RMod(short value) { return BinaryOperation(value, "%", true); } + /// + /// R Mod + /// + /// + /// public Column RMod(long value) { return BinaryOperation(value, "%", true); } + /// + /// R Mod + /// + /// + /// public Column RMod(Column value) { return BinaryOperation(value, "%", true); } + /// + /// Pow + /// + /// + /// public Column Pow(int value) { return BinaryOperation(value, "power"); } + /// + /// Pow + /// + /// + /// public Column Pow(float value) { return BinaryOperation(value, "power"); } + /// + /// Pow + /// + /// + /// public Column Pow(double value) { return BinaryOperation(value, "power"); } + /// + /// Pow + /// + /// + /// public Column Pow(short value) { return BinaryOperation(value, "power"); } + /// + /// Pow + /// + /// + /// public Column Pow(long value) { return BinaryOperation(value, "power"); } + /// + /// Pow + /// + /// + /// public Column Pow(Column value) { return BinaryOperation(value, "power"); } + /// + /// R Pow + /// + /// + /// public Column RPow(int value) { return BinaryOperation(value, "power", true); } + /// + /// R Pow + /// + /// + /// public Column RPow(float value) { return BinaryOperation(value, "power", true); } + /// + /// R Pow + /// + /// + /// public Column RPow(double value) { return BinaryOperation(value, "power", true); } + /// + /// R Pow + /// + /// + /// public Column RPow(short value) { return BinaryOperation(value, "power", true); } + /// + /// R Pow + /// + /// + /// public Column RPow(long value) { return BinaryOperation(value, "power", true); } + /// + /// R Pow + /// + /// + /// public Column RPow(Column value) { return BinaryOperation(value, "power", true); } + /// + /// Greater than + /// + /// + /// + /// public static Column operator >(Column col, int value) { return col.Gt(value); } + /// + /// Greater than + /// + /// + /// + /// public static Column operator >(Column col, float value) { return col.Gt(value); } + /// + /// Greater than + /// + /// + /// + /// public static Column operator >(Column col, double value) { return col.Gt(value); } + /// + /// Greater than + /// + /// + /// + /// public static Column operator >(Column col, short value) { return col.Gt(value); } + /// + /// Greater than + /// + /// + /// + /// public static Column operator >(Column col, long value) { return col.Gt(value); } + /// + /// Greater than + /// + /// + /// + /// public static Column operator >(Column col, Column value) { return col.Gt(value); } + /// + /// Less than + /// + /// + /// + /// public static Column operator <(Column col, int value) { return col.Lt(value); } + /// + /// Less than + /// + /// + /// + /// public static Column operator <(Column col, float value) { return col.Lt(value); } + /// + /// Less than + /// + /// + /// + /// public static Column operator <(Column col, double value) { return col.Lt(value); } + /// + /// Less than + /// + /// + /// + /// public static Column operator <(Column col, short value) { return col.Lt(value); } + /// + /// Less than + /// + /// + /// + /// public static Column operator <(Column col, long value) { return col.Lt(value); } + /// + /// Less than + /// + /// + /// + /// public static Column operator <(Column col, Column value) { return col.Lt(value); } + /// + /// Less than or equal + /// + /// + /// + /// public static Column operator <=(Column col, Column value) { return col.Le(value); } + /// + /// Less than or equal + /// + /// + /// + /// public static Column operator <=(Column col, int value) { return col.Le(value); } + /// + /// Less than or equal + /// + /// + /// + /// public static Column operator <=(Column col, float value) { return col.Le(value); } + /// + /// Less than or equal + /// + /// + /// + /// public static Column operator <=(Column col, double value) { return col.Le(value); } + /// + /// Less than or equal + /// + /// + /// + /// public static Column operator <=(Column col, short value) { return col.Le(value); } + /// + /// Less than or equal + /// + /// + /// + /// public static Column operator <=(Column col, long value) { return col.Le(value); } + /// + /// Greater than or equal + /// + /// + /// + /// public static Column operator >=(Column col, Column value) { return col.Ge(value); } + /// + /// Greater than or equal + /// + /// + /// + /// public static Column operator >=(Column col, int value) { return col.Ge(value); } + /// + /// Greater than or equal + /// + /// + /// + /// public static Column operator >=(Column col, float value) { return col.Ge(value); } + /// + /// Greater than or equal + /// + /// + /// + /// public static Column operator >=(Column col, double value) { return col.Ge(value); } + /// + /// Greater than or equal + /// + /// + /// + /// public static Column operator >=(Column col, short value) { return col.Ge(value); } + /// + /// Greater than or equal + /// + /// + /// + /// public static Column operator >=(Column col, long value) { return col.Ge(value); } + /// + /// Greater Than + /// + /// + /// public Column Gt(int value) { return BinaryOperation(value, ">"); } + /// + /// Greater Than + /// + /// + /// public Column Gt(float value) { return BinaryOperation(value, ">"); } + /// + /// Greater Than + /// + /// + /// public Column Gt(double value) { return BinaryOperation(value, ">"); } + /// + /// Greater Than + /// + /// + /// public Column Gt(short value) { return BinaryOperation(value, ">"); } + /// + /// Greater Than + /// + /// + /// public Column Gt(long value) { return BinaryOperation(value, ">"); } + /// + /// Greater Than + /// + /// + /// public Column Gt(Column value) { return BinaryOperation(value, ">"); } - + + /// + /// Less than + /// + /// + /// public Column Lt(int value) { return BinaryOperation(value, "<"); } + /// + /// Less than + /// + /// + /// public Column Lt(float value) { return BinaryOperation(value, "<"); } + /// + /// Less than + /// + /// + /// public Column Lt(double value) { return BinaryOperation(value, "<"); } + /// + /// Less than + /// + /// + /// public Column Lt(short value) { return BinaryOperation(value, "<"); } + /// + /// Less than + /// + /// + /// public Column Lt(long value) { return BinaryOperation(value, "<"); } + /// + /// Less than + /// + /// + /// public Column Lt(Column value) { return BinaryOperation(value, "<"); } - + + /// + /// Greater than or equal + /// + /// + /// public Column Ge(int value) { return BinaryOperation(value, ">="); } + /// + /// Greater than or equal + /// + /// + /// public Column Ge(float value) { return BinaryOperation(value, ">="); } + /// + /// Greater than or equal + /// + /// + /// public Column Ge(double value) { return BinaryOperation(value, ">="); } + /// + /// Greater than or equal + /// + /// + /// public Column Ge(short value) { return BinaryOperation(value, ">="); } + /// + /// Greater than or equal + /// + /// + /// public Column Ge(long value) { return BinaryOperation(value, ">="); } + /// + /// Greater than or equal + /// + /// + /// public Column Ge(Column value) { return BinaryOperation(value, ">="); } - + + /// + /// Less than or equal + /// + /// + /// public Column Le(int value) { return BinaryOperation(value, "<="); } + /// + /// Less than or equal + /// + /// + /// public Column Le(float value) { return BinaryOperation(value, "<="); } + /// + /// Less than or equal + /// + /// + /// public Column Le(double value) { return BinaryOperation(value, "<="); } + /// + /// Less than or equal + /// + /// + /// public Column Le(short value) { return BinaryOperation(value, "<="); } + /// + /// Less than or equal + /// + /// + /// public Column Le(long value) { return BinaryOperation(value, "<="); } + /// + /// Less than or equal + /// + /// + /// public Column Le(Column value) { return BinaryOperation(value, "<="); @@ -1320,11 +2342,21 @@ private Column BinaryOperation(string value, string functionName, bool reverse = return new Column(expression); } + /// + /// Over + /// + /// + /// public Column Over(WindowSpec window) { return window.ToCol(Expression); } + /// + /// Cast to the new type + /// + /// + /// Column public Column Cast(SparkDataType type) { return new Column(new Expression @@ -1336,11 +2368,20 @@ public Column Cast(SparkDataType type) }); } + /// + /// Cast to the name (string, int, etc) + /// + /// + /// public Column Cast(string type) { return Cast(SparkDataType.FromString(type)); } + /// + /// Is the column null? + /// + /// public Column IsNull() { var expression = new Expression @@ -1356,6 +2397,10 @@ public Column IsNull() return new Column(expression); } + /// + /// Is it not null? + /// + /// public Column IsNotNull() { var expression = new Expression @@ -1371,6 +2416,11 @@ public Column IsNotNull() return new Column(expression); } + /// + /// Does it end with the string other? + /// + /// + /// public Column EndsWith(string other) { var expression = new Expression @@ -1394,6 +2444,11 @@ public Column EndsWith(string other) return new Column(expression); } + /// + /// Does it start with the string other? + /// + /// + /// public Column StartsWith(string other) { var expression = new Expression @@ -1417,6 +2472,12 @@ public Column StartsWith(string other) return new Column(expression); } + /// + /// Is it between these two other columns? + /// + /// + /// + /// public Column Between(Column lowerBound, Column upperBound) { return (this >= lowerBound).And(this <= upperBound); @@ -1454,6 +2515,11 @@ public Column Otherwise(Column value) return this; } + /// + /// Drop these field names + /// + /// + /// public Column DropFields(params string[] fieldNames) { var lastExpression = Expression; @@ -1474,6 +2540,11 @@ public Column DropFields(params string[] fieldNames) return new Column(lastExpression); } + /// + /// Get a column by field name + /// + /// + /// public Column GetField(string name) { var expression = new Expression @@ -1487,6 +2558,11 @@ public Column GetField(string name) return new Column(expression); } + /// + /// Is it Like other (remember case sensitive) + /// + /// + /// public Column Like(string other) { var expression = new Expression @@ -1510,6 +2586,11 @@ public Column Like(string other) return new Column(expression); } + /// + /// Case-insensitive like + /// + /// + /// public Column ILike(string other) { var expression = new Expression @@ -1533,6 +2614,11 @@ public Column ILike(string other) return new Column(expression); } + /// + /// R Like + /// + /// + /// public Column RLike(string other) { var expression = new Expression @@ -1556,6 +2642,12 @@ public Column RLike(string other) return new Column(expression); } + /// + /// Extract the substring and return a new `Column` + /// + /// + /// + /// Column public Column Substr(int startPos, int length) { var expression = new Expression @@ -1587,11 +2679,21 @@ public Column Substr(int startPos, int length) return new Column(expression); } + /// + /// Is the value in the column in the list of values? + /// + /// + /// Column public Column IsIn(params object[] cols) { return IsIn(cols.ToList()); } + /// + /// Is the value in the column in the list of values? + /// + /// + /// Column public Column IsIn(List cols) { var expression = new Expression diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs index 3d89df6..b5e7acc 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs @@ -1548,7 +1548,7 @@ public static Column Randn(Column seed) /// /// Round /// Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// public static Column Round(string col, Column scale) { @@ -1558,7 +1558,7 @@ public static Column Round(string col, Column scale) /// /// Round /// Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// public static Column Round(Column col, Column scale) { @@ -1568,7 +1568,7 @@ public static Column Round(Column col, Column scale) /// /// Bround /// Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// public static Column Bround(string col, Column scale) { @@ -1578,7 +1578,7 @@ public static Column Bround(string col, Column scale) /// /// Bround /// Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// public static Column Bround(Column col, Column scale) { diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs deleted file mode 100644 index ce70f42..0000000 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Spark.Connect.Dotnet.Sql; - -public class CreateDataFrameHelper -{ - /// - /// Creates a `DataFrame` from a single list that contains one column - /// - /// - /// - // public LocalRelation Create(List items, string colName) - // { - // var a = new StructType(new[]{new Field("a", new Int32Type(), false, new List>())}); - // - // } - // - // public LocalRelation Create(List> rows) - // { - // - // } -} \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs index 50bd7e5..8d7b0df 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs @@ -2024,7 +2024,7 @@ public static Column Struct(params Column[] cols) /// /// Round /// Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// public static Column Round(Column col, int scale) { diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs index c803645..a607583 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs @@ -305,7 +305,7 @@ public DataFrame Sql(string sql) /// ```csharp /// var df = spark.Range(100); /// var dict = new Dictionary - /// + /// <string, object> /// (); /// dict["c"] = df["id"]; //could do Col("id") etc /// dict["dataFramePassedIn"] = df; @@ -361,7 +361,7 @@ public DataFrame Sql(string sql, params (string, DataFrame)[] dataFrames) /// /// Pass in a list of tuples, schema is guessed by the type of the first tuple's child types: /// CreateDataFrame(new List - /// + /// <object, object> /// (){ /// ("tupple", 1), ("another", 2) /// }); @@ -399,7 +399,7 @@ public DataFrame CreateDataFrame(IEnumerable<(object, object)> rows, string cola /// /// Pass in a list of tuples, schema is guessed by the type of the first tuple's child types: /// CreateDataFrame(new List - /// + /// <object, object> /// (){ /// ("tupple", 1), ("another", 2) /// }); @@ -435,7 +435,7 @@ public DataFrame CreateDataFrame(IEnumerable<(object, object)> rows) /// /// Pass in a List of Dictionary - /// - the key is used as the name of the field. The schema is guessed at using the first dictionary + /// <string, type> - the key is used as the name of the field. The schema is guessed at using the first dictionary /// /// /// @@ -455,7 +455,7 @@ public DataFrame CreateDataFrame(IEnumerable> rows) /// /// Pass in a List of Dictionary - /// - the key is used as the name of the field. The schema is passed in explicitly + /// <string, type> - the key is used as the name of the field. The schema is passed in explicitly /// /// /// @@ -519,16 +519,16 @@ public DataFrame CreateDataFrame(IEnumerable> data, params s /// /// Pass in rows and an explicit schema: /// CreateDataFrame(new List - /// + /// <object> /// (){ /// new List - /// + /// <object> /// (){"abc", 123, 100.123}, /// new List - /// + /// <object> /// (){"def", 456, 200.456}, /// new List - /// + /// <object> /// (){"xyz", 999, 999.456}, /// }, new StructType( /// new StructField("col_a", StringType(), true),