From 0ad5e00b9112529707fc86a086bb0265734c70a1 Mon Sep 17 00:00:00 2001 From: Ed Elliott <GoEddie@users.noreply.github.com> Date: Wed, 4 Dec 2024 19:56:35 +0000 Subject: [PATCH] code function docs (#47) --- .../Databricks/IsDatabricks.cs | 8 + .../Spark.Connect.Dotnet/Grpc/GrpcInternal.cs | 102 ++ .../Spark.Connect.Dotnet/Grpc/GrpcLogger.cs | 50 + .../Spark.Connect.Dotnet/Grpc/Logger.cs | 16 + .../Grpc/RequestExecutor.cs | 83 +- ...taTypeMismatchNonFoldableInputException.cs | 12 + ...ypeMismatchUnexpectedInputTypeException.cs | 12 + .../GroupByAggregateException.cs | 13 + .../SparkExceptions/InternalSparkException.cs | 12 + .../MissingGroupByException.cs | 12 + .../Grpc/SparkExceptions/SparkException.cs | 23 + .../SparkExceptions/SparkExceptionFactory.cs | 13 + .../SparkExceptions/UnavailableException.cs | 22 + ...UnresolvedColumnWithSuggestionException.cs | 35 +- .../UnresolvedRoutineException.cs | 21 +- .../WindowFunctionWithoutOverException.cs | 38 + .../Spark.Connect.Dotnet/LocalConsole.cs | 11 + .../Spark.Connect.Dotnet/Sql/Column.cs | 1122 ++++++++++++++++- .../Spark.Connect.Dotnet/Sql/Functions.cs | 8 +- .../Sql/Internal/CreateDataFrameHelper.cs | 20 - .../Sql/ManualFunctions.cs | 2 +- .../Spark.Connect.Dotnet/Sql/SparkSession.cs | 18 +- 22 files changed, 1558 insertions(+), 95 deletions(-) create mode 100644 src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs create mode 100644 src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs delete mode 100644 src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs index 4ce2040..4e224ea 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Databricks/IsDatabricks.cs @@ -1,7 +1,15 @@ namespace Spark.Connect.Dotnet.Databricks; +/// <summary> +/// Are we connecting to Databricks? used to control whether we wait for a connection. +/// </summary> public class IsDatabricks { + /// <summary> + /// Valiudates whether the url contains the name 'databricks' + /// </summary> + /// <param name="url"></param> + /// <returns></returns> public static bool Url(string url) { return url.Contains("databricks", StringComparison.OrdinalIgnoreCase); diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs index 68764f5..964f700 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcInternal.cs @@ -7,8 +7,23 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// This is the class used to pass messages down the gRPC channel. +/// </summary> public static class GrpcInternal { + /// <summary> + /// Explain + /// </summary> + /// <param name="client"></param> + /// <param name="sessionId"></param> + /// <param name="plan"></param> + /// <param name="headers"></param> + /// <param name="userContext"></param> + /// <param name="clientType"></param> + /// <param name="explainExtended"></param> + /// <param name="mode"></param> + /// <returns></returns> public static string Explain(SparkConnectService.SparkConnectServiceClient client, string sessionId, Plan plan, Metadata headers, UserContext userContext, string clientType, bool explainExtended, string? mode) { @@ -34,6 +49,13 @@ public static string Explain(SparkConnectService.SparkConnectServiceClient clien return analyzeResponse.Explain.ExplainString; } + /// <summary> + /// Persist + /// </summary> + /// <param name="session"></param> + /// <param name="relation"></param> + /// <param name="storageLevel"></param> + /// <returns></returns> public static Relation Persist(SparkSession session, Relation relation, StorageLevel storageLevel) { var analyzeRequest = new AnalyzePlanRequest @@ -49,6 +71,18 @@ public static Relation Persist(SparkSession session, Relation relation, StorageL return relation; } + /// <summary> + /// Schema + /// </summary> + /// <param name="client"></param> + /// <param name="sessionId"></param> + /// <param name="plan"></param> + /// <param name="headers"></param> + /// <param name="userContext"></param> + /// <param name="clientType"></param> + /// <param name="explainExtended"></param> + /// <param name="mode"></param> + /// <returns></returns> public static DataType Schema(SparkConnectService.SparkConnectServiceClient client, string sessionId, Plan plan, Metadata headers, UserContext userContext, string clientType, bool explainExtended, string mode) { @@ -65,6 +99,11 @@ public static DataType Schema(SparkConnectService.SparkConnectServiceClient clie return analyzeResponse.Schema.Schema_; } + /// <summary> + /// What is the Spark Version you are connect to? + /// </summary> + /// <param name="session"></param> + /// <returns></returns> public static string Version(SparkSession session) { var analyzeRequest = new AnalyzePlanRequest @@ -76,6 +115,12 @@ public static string Version(SparkSession session) return analyzeResponse.SparkVersion.Version; } + /// <summary> + /// Get a list of the input files + /// </summary> + /// <param name="session"></param> + /// <param name="plan"></param> + /// <returns></returns> public static IEnumerable<string> InputFiles(SparkSession session, Plan plan) { var analyzeRequest = new AnalyzePlanRequest @@ -91,6 +136,12 @@ public static IEnumerable<string> InputFiles(SparkSession session, Plan plan) return analyzeResponse.InputFiles.Files.Select(p => p); } + /// <summary> + /// Is it a local plan? + /// </summary> + /// <param name="session"></param> + /// <param name="plan"></param> + /// <returns></returns> public static bool IsLocal(SparkSession session, Plan plan) { var analyzeRequest = new AnalyzePlanRequest @@ -106,6 +157,13 @@ public static bool IsLocal(SparkSession session, Plan plan) return analyzeResponse.IsLocal.IsLocal_; } + /// <summary> + /// Get the TreeString + /// </summary> + /// <param name="session"></param> + /// <param name="relation"></param> + /// <param name="level"></param> + /// <returns></returns> public static string TreeString(SparkSession session, Relation relation, int? level = null) { var analyzeRequest = new AnalyzePlanRequest @@ -129,6 +187,12 @@ public static string TreeString(SparkSession session, Relation relation, int? le return analyzeResponse.TreeString.TreeString_; } + /// <summary> + /// Create a semantic hash of the relation + /// </summary> + /// <param name="session"></param> + /// <param name="relation"></param> + /// <returns></returns> public static int SemanticHash(SparkSession session, Relation relation) { var analyzeRequest = new AnalyzePlanRequest @@ -148,6 +212,12 @@ public static int SemanticHash(SparkSession session, Relation relation) return analyzeResponse.SemanticHash.Result; } + /// <summary> + /// What is the storage level? + /// </summary> + /// <param name="session"></param> + /// <param name="relation"></param> + /// <returns></returns> public static StorageLevel StorageLevel(SparkSession session, Relation relation) { var analyzeRequest = new AnalyzePlanRequest @@ -164,6 +234,12 @@ public static StorageLevel StorageLevel(SparkSession session, Relation relation) return analyzeResponse.GetStorageLevel.StorageLevel; } + /// <summary> + /// Is it a Streaming plan + /// </summary> + /// <param name="session"></param> + /// <param name="plan"></param> + /// <returns></returns> public static bool IsStreaming(SparkSession session, Plan plan) { var analyzeRequest = new AnalyzePlanRequest @@ -179,6 +255,13 @@ public static bool IsStreaming(SparkSession session, Plan plan) return analyzeResponse.IsStreaming.IsStreaming_; } + /// <summary> + /// Same Semantics, uses AnalyzePlanRequest + /// </summary> + /// <param name="session"></param> + /// <param name="target"></param> + /// <param name="other"></param> + /// <returns></returns> public static bool SameSemantics(SparkSession session, Relation target, Relation other) { var analyzeRequest = new AnalyzePlanRequest @@ -201,6 +284,12 @@ public static bool SameSemantics(SparkSession session, Relation target, Relation return analyzeResponse.SameSemantics.Result; } + /// <summary> + /// Unset a config option + /// </summary> + /// <param name="session"></param> + /// <param name="key"></param> + /// <exception cref="SparkException"></exception> public static async Task ExecUnSetConfigCommandResponse(SparkSession session, string key) { var configRequest = new ConfigRequest @@ -245,6 +334,12 @@ AsyncUnaryCall<ConfigResponse> Exec() } } + /// <summary> + /// Set Config Item + /// </summary> + /// <param name="session"></param> + /// <param name="options"></param> + /// <exception cref="SparkException"></exception> public static async Task ExecSetConfigCommandResponse(SparkSession session, IDictionary<string, string> options) { var configRequest = new ConfigRequest @@ -295,6 +390,13 @@ AsyncUnaryCall<ConfigResponse> Exec() } } + /// <summary> + /// Get All Config Response + /// </summary> + /// <param name="session"></param> + /// <param name="prefix"></param> + /// <returns></returns> + /// <exception cref="SparkException"></exception> public static async Task<Dictionary<string, string>> ExecGetAllConfigCommandResponse(SparkSession session, string? prefix = null) { diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs index 7250a37..a0abb03 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/GrpcLogger.cs @@ -1,10 +1,18 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Used to log gRPC info, needs migrating to a better logger. +/// </summary> public class GrpcLogger { private readonly GrpcLoggingLevel _level; private readonly LocalConsole _console; + /// <summary> + /// Create a logger, pass in your own console if you want to redirect output (like in tests!) + /// </summary> + /// <param name="level"></param> + /// <param name="console"></param> public GrpcLogger(GrpcLoggingLevel level, LocalConsole? console = null) { _level = level; @@ -13,6 +21,11 @@ public GrpcLogger(GrpcLoggingLevel level, LocalConsole? console = null) } + /// <summary> + /// Log + /// </summary> + /// <param name="level"></param> + /// <param name="message"></param> public virtual void Log(GrpcLoggingLevel level, string message) { if (level >= _level) @@ -21,6 +34,12 @@ public virtual void Log(GrpcLoggingLevel level, string message) } } + /// <summary> + /// Log + /// </summary> + /// <param name="level"></param> + /// <param name="format"></param> + /// <param name="args"></param> public virtual void Log(GrpcLoggingLevel level, string format, params object[] args) { if (level >= _level) @@ -30,26 +49,57 @@ public virtual void Log(GrpcLoggingLevel level, string format, params object[] a } } +/// <summary> +/// The default logger that does nothing +/// </summary> public class GrpcNullLogger : GrpcLogger { + /// <summary> + /// Create the default null logger + /// </summary> + /// <param name="level"></param> + /// <param name="console"></param> public GrpcNullLogger(GrpcLoggingLevel level, LocalConsole? console = null) : base(level, console) { } + /// <summary> + /// Log + /// </summary> + /// <param name="level"></param> + /// <param name="format"></param> + /// <param name="args"></param> public override void Log(GrpcLoggingLevel level, string format, params object[] args) { } + /// <summary> + /// Log + /// </summary> + /// <param name="level"></param> + /// <param name="message"></param> public override void Log(GrpcLoggingLevel level, string message) { } } +/// <summary> +/// Log Level +/// </summary> public enum GrpcLoggingLevel { + /// <summary> + /// None + /// </summary> None, + /// <summary> + /// Warn + /// </summary> Warn, + /// <summary> + /// Verbose + /// </summary> Verbose } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs index 74b25a7..2b17354 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/Logger.cs @@ -1,14 +1,25 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Logger +/// </summary> public class Logger { private static int _level; + /// <summary> + /// Create the logger at whatever level you like + /// </summary> + /// <param name="level"></param> public Logger(int level) { _level = level; } + /// <summary> + /// Log + /// </summary> + /// <param name="message"></param> public static void WriteLine(string message) { if (_level > 1) @@ -17,6 +28,11 @@ public static void WriteLine(string message) } } + /// <summary> + /// Log + /// </summary> + /// <param name="message"></param> + /// <param name="parameters"></param> public static void WriteLine(string message, params object[] parameters) { if (_level > 1) diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs index ff7384c..689e371 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/RequestExecutor.cs @@ -4,21 +4,24 @@ namespace Spark.Connect.Dotnet.Grpc; -/* - * When we connect to remote spark clusters, including Databricks we can find our tcp connections are closed - * Azure has a 5 minute idle time where connections are killed and Databricks has a hard 1 hour timeout so we - * need to create the initial request with the Reattach option Reattachable set to true and then we can re-connect - * a failed connection using ReattachExecute so we don't have to re-run the query from the beginning. - * - * The .NET gRPC request will sit forever and not respond if the connection is killed so we use a scheduled cancellation - * to cancel the request if we haven't had a response in 1 minute. - * - * When we get the first response it includes an operation id and a request id - we use the operation id to identify the query run - * and the response id to track the responses we have received. the response id is like a pointer to tell the server where we have - * received the response up to. - * - * When we finish and have everything we need we should tell the server to release the responses so the memory is freed on the server. - */ + + + +/// <summary> +/// When we connect to remote spark clusters, including Databricks we can find our tcp connections are closed +/// Azure has a 5 minute idle time where connections are killed and Databricks has a hard 1 hour timeout so we +/// need to create the initial request with the Reattach option Reattachable set to true and then we can re-connect +/// a failed connection using ReattachExecute so we don't have to re-run the query from the beginning. +/// +/// The .NET gRPC request will sit forever and not respond if the connection is killed so we use a scheduled cancellation +/// to cancel the request if we haven't had a response in 1 minute. +/// +/// When we get the first response it includes an operation id and a request id - we use the operation id to identify the query run +/// and the response id to track the responses we have received. the response id is like a pointer to tell the server where we have +/// received the response up to. +/// +/// When we finish and have everything we need we should tell the server to release the responses so the memory is freed on the server. +/// </summary> public class RequestExecutor : IDisposable { private readonly SparkSession _session; @@ -49,6 +52,11 @@ private enum RetryableState private RetryableState _retryableState = RetryableState.Processing; + /// <summary> + /// Create the Executor + /// </summary> + /// <param name="session"></param> + /// <param name="plan"></param> public RequestExecutor(SparkSession session, Plan plan) { _logger = GetLogger(session); @@ -71,12 +79,18 @@ private GrpcLogger GetLogger(SparkSession session) return new GrpcNullLogger(GrpcLoggingLevel.None, null); } + /// <summary> + /// Excecute the plan passed in the constructor + /// </summary> public void Exec() { var task = Task.Run(ExecAsync); task.Wait(); } + /// <summary> + /// Execute the plan already passed into the constructor + /// </summary> public async Task ExecAsync() { var shouldContinue = true; @@ -353,6 +367,9 @@ private void PrintObservedMetrics(RepeatedField<ExecutePlanResponse.Types.Observ } + /// <summary> + /// Dispose releasing any outstanding requests + /// </summary> public void Dispose() { if (_operationId != string.Empty && _lastResponseId != String.Empty) @@ -367,22 +384,58 @@ public void Dispose() } } + /// <summary> + /// Get any data returned by the last request + /// </summary> + /// <returns></returns> public IList<Row> GetData() => _rows; + /// <summary> + /// The schema of the last request (if available) + /// </summary> + /// <returns></returns> public DataType GetSchema() => _schema!; + /// <summary> + /// If there was a relation returned + /// </summary> + /// <returns></returns> public Relation GetRelation() => _relation!; + /// <summary> + /// Get the streaming query id + /// </summary> + /// <returns></returns> public StreamingQueryInstanceId GetStreamingQueryId() => _streamingQueryId!; + /// <summary> + /// Streaming command result + /// </summary> + /// <returns></returns> public StreamingQueryCommandResult.Types.StatusResult GetStreamingQueryCommandResult() => _streamingResultStatus!; + /// <summary> + /// Streaming query name + /// </summary> + /// <returns></returns> public string GetStreamingQueryName() => _streamingQueryName!; + /// <summary> + /// Is the streaming query terminated? + /// </summary> + /// <returns></returns> public bool GetStreamingQueryIsTerminated() => _streamingQueryIsTerminated!.Value; + /// <summary> + /// Any streaming exceptions + /// </summary> + /// <returns></returns> public StreamingQueryCommandResult.Types.ExceptionResult? GetStreamingException() => _streamingQueryException; + /// <summary> + /// Any streaming progress info + /// </summary> + /// <returns></returns> public StreamingQueryCommandResult.Types.RecentProgressResult? GetStreamingRecentProgress() => _streamingProgress; } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs index 5708718..3e6f86e 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchNonFoldableInputException.cs @@ -1,13 +1,25 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Wrap DATATYPE_MISMATCH.NON_FOLDABLE_INPUT +/// </summary> public class DataTypeMismatchNonFoldableInputException : SparkException { //DATATYPE_MISMATCH.NON_FOLDABLE_INPUT + /// <summary> + /// Wrap DATATYPE_MISMATCH.NON_FOLDABLE_INPUT + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public DataTypeMismatchNonFoldableInputException(string exceptionMessage, Exception exception) : base( exceptionMessage, exception) { } + /// <summary> + /// Wrap DATATYPE_MISMATCH.NON_FOLDABLE_INPUT + /// </summary> + /// <param name="exception"></param> public DataTypeMismatchNonFoldableInputException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs index 5fa43ba..681be41 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/DataTypeMismatchUnexpectedInputTypeException.cs @@ -1,7 +1,15 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Wrap [DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] +/// </summary> public class DataTypeMismatchUnexpectedInputTypeException : SparkException { + /// <summary> + /// Wrap [DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public DataTypeMismatchUnexpectedInputTypeException(string exceptionMessage, Exception exception) : base( exceptionMessage, exception) { @@ -9,6 +17,10 @@ public DataTypeMismatchUnexpectedInputTypeException(string exceptionMessage, Exc OverrideMessage = exceptionMessage.Replace("[DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE]", ""); } + /// <summary> + /// Wrap [DATATYPE_MISMATCH.UNEXPECTED_INPUT_TYPE] + /// </summary> + /// <param name="exception"></param> public DataTypeMismatchUnexpectedInputTypeException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs index 893412d..cb408b4 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/GroupByAggregateException.cs @@ -1,12 +1,25 @@ namespace Spark.Connect.Dotnet.Grpc; + +/// <summary> +/// Wrap GROUP_BY_AGGREGATE +/// </summary> public class GroupByAggregateException : SparkException { //GROUP_BY_AGGREGATE + /// <summary> + /// Wrap GROUP_BY_AGGREGATE + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public GroupByAggregateException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } + /// <summary> + /// Wrap GROUP_BY_AGGREGATE + /// </summary> + /// <param name="exception"></param> public GroupByAggregateException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs index 58fafd3..ac6aacb 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/InternalSparkException.cs @@ -1,12 +1,24 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Generic internal error +/// </summary> public class InternalSparkException : SparkException { //Internal + /// <summary> + /// Generic internal error + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public InternalSparkException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } + /// <summary> + /// Generic internal error + /// </summary> + /// <param name="exception"></param> public InternalSparkException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs index f8f16b1..70f76ec 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/MissingGroupByException.cs @@ -1,11 +1,23 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Wrap MISSING_GROUP_BY +/// </summary> public class MissingGroupByException : SparkException { + /// <summary> + /// Wrap MISSING_GROUP_BY + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public MissingGroupByException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } + /// <summary> + /// Wrap MISSING_GROUP_BY + /// </summary> + /// <param name="exception"></param> public MissingGroupByException(Exception exception) : base(exception) { } diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs index 7f1ac4f..681f352 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkException.cs @@ -2,10 +2,22 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Helper exceptions for any errors we get +/// </summary> public class SparkException : Exception { + /// <summary> + /// Implementing exceptions can override the default error message + /// </summary> protected string OverrideMessage = ""; + + /// <summary> + /// Helper exceptions for any errors we get + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public SparkException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { if (exception is AggregateException && exception.InnerException is RpcException rpcException) @@ -19,6 +31,10 @@ public SparkException(string exceptionMessage, Exception exception) : base(excep } } + /// <summary> + /// Helper exceptions for any errors we get + /// </summary> + /// <param name="exception"></param> public SparkException(Exception exception) : base(exception.Message, exception) { if (exception is AggregateException && exception.InnerException is RpcException rpcException) @@ -27,10 +43,17 @@ public SparkException(Exception exception) : base(exception.Message, exception) } } + /// <summary> + /// Helper exceptions for any errors we get + /// </summary> + /// <param name="message"></param> public SparkException(string message) : base(message) { OverrideMessage = message; } + /// <summary> + /// Error message + /// </summary> public override string Message => OverrideMessage; } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs index 4c5eac9..c0256bf 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/SparkExceptionFactory.cs @@ -2,6 +2,9 @@ namespace Spark.Connect.Dotnet.Grpc.SparkExceptions; +/// <summary> +/// Used to help map and create exceptions +/// </summary> public static class SparkExceptionFactory { private static SparkException DetailStringToException(string exceptionCode, string detail, Exception exception) @@ -32,6 +35,11 @@ private static string ExceptionCodeFromSparkError(string detail, string statusCo return exceptionCode; } + /// <summary> + /// Create the appropriate exception + /// </summary> + /// <param name="exception"></param> + /// <returns></returns> public static SparkException GetExceptionFromRpcException(RpcException exception) { var exceptionCode = @@ -39,6 +47,11 @@ public static SparkException GetExceptionFromRpcException(RpcException exception return DetailStringToException(exceptionCode, exception.Status.Detail, exception); } + /// <summary> + /// Create the appropriate exception + /// </summary> + /// <param name="aggException"></param> + /// <returns></returns> public static SparkException GetExceptionFromRpcException(AggregateException aggException) { if (aggException.InnerException is RpcException exception) diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs new file mode 100644 index 0000000..689c186 --- /dev/null +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnavailableException.cs @@ -0,0 +1,22 @@ +namespace Spark.Connect.Dotnet.Grpc; + +/// <summary> +/// Wrap Unavailable +/// </summary> +public class UnavailableException : SparkException +{ + //"Unavailable" + + /// <summary> + /// Wrap Unavailable + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> + public UnavailableException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) + { + if (exceptionMessage == "Error connecting to subchannel.") + { + OverrideMessage = "Cannot connect to remote Spark Server, if your $SPARK_REMOTE set correctly?"; + } + } +} \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs index d6921a7..f8fcb0e 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedColumnWithSuggestionException.cs @@ -1,12 +1,19 @@ using System.Text.RegularExpressions; -using Spark.Connect.Dotnet.Sql; namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Wrap UNRESOLVED_COLUMN.WITH_SUGGESTION +/// </summary> public class UnresolvedColumnWithSuggestionException : SparkException { //UNRESOLVED_COLUMN.WITH_SUGGESTION + /// <summary> + /// Wrap UNRESOLVED_COLUMN.WITH_SUGGESTION + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public UnresolvedColumnWithSuggestionException(string exceptionMessage, Exception exception) : base( exceptionMessage, exception) { @@ -25,29 +32,11 @@ public UnresolvedColumnWithSuggestionException(string exceptionMessage, Exceptio OverrideMessage = $@"The Column `{columnName}` was not found, did you mean one of: {existingColumns}?"; } + /// <summary> + /// Wrap UNRESOLVED_COLUMN.WITH_SUGGESTION + /// </summary> + /// <param name="exception"></param> public UnresolvedColumnWithSuggestionException(Exception exception) : base(exception) { } -} - -public class WindowFunctionWithoutOverException : SparkException -{ - //WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE - - public WindowFunctionWithoutOverException(string exceptionMessage, Exception exception) : base(exceptionMessage, - exception) - { - //Format = [WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE] Window function "(.*?)" requires an OVER clause. - var input = exceptionMessage; - // Extract the column name - var windowFunctionPattern = @"Window function ""(.*?)"""; - var windowFunction = Regex.Match(input, windowFunctionPattern).Value; - - OverrideMessage = - $@"The Window Function `{FunctionsWrapper.CSharpFunctionName(windowFunction)}` requires an OVER clause"; - } - - public WindowFunctionWithoutOverException(Exception exception) : base(exception) - { - } } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs index 6585c53..bf281ba 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/UnresolvedRoutineException.cs @@ -1,23 +1,18 @@ namespace Spark.Connect.Dotnet.Grpc; +/// <summary> +/// Wrap UNRESOLVED_ROUTINE +/// </summary> public class UnresolvedRoutineException : SparkException { //"UNRESOLVED_ROUTINE" + /// <summary> + /// Wrap UNRESOLVED_ROUTINE + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> public UnresolvedRoutineException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) { } -} - -public class UnavailableException : SparkException -{ - //"Unavailable" - - public UnavailableException(string exceptionMessage, Exception exception) : base(exceptionMessage, exception) - { - if (exceptionMessage == "Error connecting to subchannel.") - { - OverrideMessage = "Cannot connect to remote Spark Server, if your $SPARK_REMOTE set correctly?"; - } - } } \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs new file mode 100644 index 0000000..d5051ca --- /dev/null +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Grpc/SparkExceptions/WindowFunctionWithoutOverException.cs @@ -0,0 +1,38 @@ +using System.Text.RegularExpressions; +using Spark.Connect.Dotnet.Sql; + +namespace Spark.Connect.Dotnet.Grpc; + +/// <summary> +/// Wrap WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE +/// </summary> +public class WindowFunctionWithoutOverException : SparkException +{ + //WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE + + /// <summary> + /// Wrap WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE + /// </summary> + /// <param name="exceptionMessage"></param> + /// <param name="exception"></param> + public WindowFunctionWithoutOverException(string exceptionMessage, Exception exception) : base(exceptionMessage, + exception) + { + //Format = [WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE] Window function "(.*?)" requires an OVER clause. + var input = exceptionMessage; + // Extract the column name + var windowFunctionPattern = @"Window function ""(.*?)"""; + var windowFunction = Regex.Match(input, windowFunctionPattern).Value; + + OverrideMessage = + $@"The Window Function `{FunctionsWrapper.CSharpFunctionName(windowFunction)}` requires an OVER clause"; + } + + /// <summary> + /// Wrap WINDOW_FUNCTION_WITHOUT_OVER_CLAUSE + /// </summary> + /// <param name="exception"></param> + public WindowFunctionWithoutOverException(Exception exception) : base(exception) + { + } +} \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs index c71b4f1..559da63 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/LocalConsole.cs @@ -1,12 +1,23 @@ namespace Spark.Connect.Dotnet; +/// <summary> +/// This is used by our tests to redirect stdout to the console +/// </summary> public class LocalConsole { + /// <summary> + /// Write to the console + /// </summary> + /// <param name="what"></param> public virtual void Write(string what) { Console.Write(what); } + /// <summary> + /// WriteLine to the console + /// </summary> + /// <param name="what"></param> public virtual void WriteLine(string what) { Console.WriteLine(what); diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs index 36ada7b..997f6ff 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Column.cs @@ -2,21 +2,44 @@ namespace Spark.Connect.Dotnet.Sql; +/// <summary> +/// A column in a DataFrame. +/// +/// This wraps an `Expression` which is what we pass to Spark Connect. +/// </summary> public class Column { private readonly string _name = string.Empty; + + /// <summary> + /// This is what the Column actually does, it is what is passed to spark + /// </summary> public readonly Expression Expression; + /// <summary> + /// Get a column by name. Static version used by the functions and wraps `Column(name)` + /// </summary> + /// <param name="name"></param> + /// <returns>Column</returns> public static Column Col(string name) { return new Column(name); } + /// <summary> + /// Create a `Column` from an `Expression` + /// </summary> + /// <param name="expression"></param> public Column(Expression expression) { Expression = expression; } + /// <summary> + /// Pass either the name of a column, or *. You can also pass a `DataFrame` to use as the source. + /// </summary> + /// <param name="name"></param> + /// <param name="source"></param> public Column(string name, DataFrame? source = null) { _name = name; @@ -72,6 +95,12 @@ private bool Equals(Column other) return Expression.Equals(other.Expression); } + + /// <summary> + /// Does this column equal the passed in object? + /// </summary> + /// <param name="obj"></param> + /// <returns></returns> public override bool Equals(object? obj) { if (ReferenceEquals(null, obj)) @@ -87,11 +116,20 @@ public override bool Equals(object? obj) return obj.GetType() == GetType() && Equals((Column)obj); } + /// <summary> + /// Get the hash code of the column (hopefully no one needs this, if they do then it sounds like a debugging nightmare) + /// </summary> + /// <returns></returns> public override int GetHashCode() { return Expression.GetHashCode(); } + /// <summary> + /// Alias the column to another name + /// </summary> + /// <param name="name"></param> + /// <returns>Column</returns> public Column Alias(string name) { var expression = new Expression @@ -105,11 +143,20 @@ public Column Alias(string name) return new Column(expression); } + /// <summary> + /// Alias the column to another name + /// </summary> + /// <param name="name">The new name</param> + /// <returns>Column</returns> public Column As(string name) { return Alias(name); } + /// <summary> + /// Order the existing column ascending + /// </summary> + /// <returns></returns> public Column Asc() { return new Column(new Expression @@ -121,6 +168,10 @@ public Column Asc() }); } + /// <summary> + /// Order the existing column descending + /// </summary> + /// <returns></returns> public Column Desc() { return new Column(new Expression @@ -132,909 +183,1880 @@ public Column Desc() }); } + /// <summary> + /// & the column with the passed in bool + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator &(Column src, bool value) { return src.And(value); } + /// <summary> + /// & the column with the passed in `Column` + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator &(Column src, Column value) { return src.And(value); } + /// <summary> + /// & the column with the passed in bool + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column And(bool value) { return BinaryOperation(value, "and"); } + /// <summary> + /// & the column with the passed in `Column` + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column And(Column value) { return BinaryOperation(value, "and"); } + /// <summary> + /// | the column with the passed in bool + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator |(Column src, bool value) { return src.Or(value); } + /// <summary> + /// | the column with the passed in `Column` + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator |(Column src, Column value) { return src.Or(value); } + /// <summary> + /// | the column with the passed in bool + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Or(bool value) { return BinaryOperation(value, "or"); } + /// <summary> + /// | the column with the passed in `Column` + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Or(Column value) { return BinaryOperation(value, "or"); } + /// <summary> + /// ! the column + /// </summary> + /// <param name="src"></param> + /// <returns></returns> public static Column operator !(Column src) { return src.Not(); } + /// <summary> + /// ! the column + /// </summary> + /// <returns></returns> public Column Not() { return BinaryOperation("not"); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator *(Column src, int value) { return src.Multiply(value); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="src"></param> + /// <param name="right"></param> + /// <returns></returns> public static Column operator *(Column src, double right) { return src.Multiply(right); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="src"></param> + /// <param name="right"></param> + /// <returns></returns> public static Column operator *(Column src, float right) { return src.Multiply(right); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="src"></param> + /// <param name="right"></param> + /// <returns></returns> public static Column operator *(Column src, long right) { return src.Multiply(right); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="src"></param> + /// <param name="right"></param> + /// <returns></returns> public static Column operator *(Column src, short right) { return src.Multiply(right); } + /// <summary> + /// Multiply the column with the other column + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator *(Column src, Column value) { return src.Multiply(value); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Multiply(int value) { return BinaryOperation(value, "*"); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Multiply(float value) { return BinaryOperation(value, "*"); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Multiply(double value) { return BinaryOperation(value, "*"); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Multiply(long value) { return BinaryOperation(value, "*"); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Multiply(short value) { return BinaryOperation(value, "*"); } + /// <summary> + /// Multiply the column with the value + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Multiply(Column value) { return BinaryOperation(value, "*"); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, int value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, float value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, double value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, bool value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, long value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, string value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator ==(Column src, Column value) { return src.EqualTo(value); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(string value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(int value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(float value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(double value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(bool value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(long value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column EqualTo(Column value) { return BinaryOperation(value, "=="); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, int value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, string value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, long value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, float value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, double value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, bool value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator !=(Column src, Column value) { return src.NotEqualTo(value); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(int value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(string value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(float value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(double value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(bool value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(long value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// Does the column not equal the other one? + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column NotEqualTo(Column value) { var equals = BinaryOperation(value, "=="); return NotOperation(equals); } + /// <summary> + /// RMultiply by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMultiply(int value) { return BinaryOperation(value, "*", true); } + /// <summary> + /// RMultiply by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMultiply(Column value) { return BinaryOperation(value, "*", true); } - + + /// <summary> + /// Divide by + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator /(Column src, int value) { return src.Divide(value); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator /(Column src, float value) { return src.Divide(value); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator /(Column src, double value) { return src.Divide(value); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator /(Column src, short value) { return src.Divide(value); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator /(Column src, long value) { return src.Divide(value); } - public static Column operator /(Column src, Column value) - { - return src.Divide(value); + /// <summary> + /// Divide by + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> + public static Column operator /(Column src, Column value) + { + return src.Divide(value); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Divide(int value) { return BinaryOperation(value, "/"); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Divide(float value) { return BinaryOperation(value, "/"); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Divide(double value) { return BinaryOperation(value, "/"); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Divide(short value) { return BinaryOperation(value, "/"); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Divide(long value) { return BinaryOperation(value, "/"); } + /// <summary> + /// Divide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Divide(Column value) { return BinaryOperation(value, "/"); } + /// <summary> + /// RDivide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RDivide(int value) { return BinaryOperation(value, "/", true); } + /// <summary> + /// RDivide by + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RDivide(Column value) { return BinaryOperation(value, "/", true); } + /// <summary> + /// Add + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator +(Column src, int value) { return src.Add(value); } + /// <summary> + /// Add + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator +(Column src, float value) { return src.Add(value); } + /// <summary> + /// Add + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator +(Column src, double value) { return src.Add(value); } + /// <summary> + /// Add + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator +(Column src, short value) { return src.Add(value); } + /// <summary> + /// Add + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator +(Column src, long value) { return src.Add(value); } + /// <summary> + /// Add + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator +(Column src, Column value) { return src.Add(value); } + /// <summary> + /// Add + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Add(int value) { return BinaryOperation(value, "+"); } + /// <summary> + /// Add + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Add(float value) { return BinaryOperation(value, "+"); } + /// <summary> + /// Add + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Add(double value) { return BinaryOperation(value, "+"); } + /// <summary> + /// Add + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Add(short value) { return BinaryOperation(value, "+"); } + /// <summary> + /// Add + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Add(long value) { return BinaryOperation(value, "+"); } + /// <summary> + /// Add + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Add(Column value) { return BinaryOperation(value, "+"); } - + + /// <summary> + /// RAdd + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RAdd(int value) { return BinaryOperation(value, "+", true); } + /// <summary> + /// RAdd + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RAdd(float value) { return BinaryOperation(value, "+", true); } + /// <summary> + /// RAdd + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RAdd(double value) { return BinaryOperation(value, "+", true); } + /// <summary> + /// RAdd + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RAdd(short value) { return BinaryOperation(value, "+", true); } + /// <summary> + /// RAdd + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RAdd(long value) { return BinaryOperation(value, "+", true); } + /// <summary> + /// RAdd + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RAdd(Column value) { return BinaryOperation(value, "+", true); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator -(Column src, int value) { return src.Minus(value); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator -(Column src, float value) { return src.Minus(value); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator -(Column src, double value) { return src.Minus(value); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator -(Column src, short value) { return src.Minus(value); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator -(Column src, long value) { return src.Minus(value); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator -(Column src, Column value) { return src.Minus(value); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Minus(int value) { return BinaryOperation(value, "-"); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Minus(float value) { return BinaryOperation(value, "-"); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Minus(double value) { return BinaryOperation(value, "-"); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Minus(short value) { return BinaryOperation(value, "-"); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Minus(long value) { return BinaryOperation(value, "-"); } + /// <summary> + /// Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Minus(Column value) { return BinaryOperation(value, "-"); } + /// <summary> + /// R Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMinus(int value) { return BinaryOperation(value, "-", true); } + /// <summary> + /// R Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMinus(float value) { return BinaryOperation(value, "-", true); } - + + /// <summary> + /// R Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMinus(double value) { return BinaryOperation(value, "-", true); } + /// <summary> + /// R Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMinus(short value) { return BinaryOperation(value, "-", true); } + /// <summary> + /// R Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMinus(long value) { return BinaryOperation(value, "-", true); } + /// <summary> + /// R Subtract + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMinus(Column value) { return BinaryOperation(value, "-", true); } - + + /// <summary> + /// Mod + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator %(Column src, int value) { return src.Mod(value); } + /// <summary> + /// Mod + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator %(Column src, float value) { return src.Mod(value); } + /// <summary> + /// Mod + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator %(Column src, double value) { return src.Mod(value); } + /// <summary> + /// Mod + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator %(Column src, short value) { return src.Mod(value); } + /// <summary> + /// Mod + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator %(Column src, long value) { return src.Mod(value); } + /// <summary> + /// Mod + /// </summary> + /// <param name="src"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator %(Column src, Column value) { return src.Mod(value); } + /// <summary> + /// Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Mod(int value) { return BinaryOperation(value, "%"); } + /// <summary> + /// Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Mod(float value) { return BinaryOperation(value, "%"); } + /// <summary> + /// Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Mod(double value) { return BinaryOperation(value, "%"); } + /// <summary> + /// Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Mod(short value) { return BinaryOperation(value, "%"); } + /// <summary> + /// Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Mod(long value) { return BinaryOperation(value, "%"); } + /// <summary> + /// Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Mod(Column value) { return BinaryOperation(value, "%"); } + /// <summary> + /// R Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMod(int value) { return BinaryOperation(value, "%", true); } + /// <summary> + /// R Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMod(float value) { return BinaryOperation(value, "%", true); } + /// <summary> + /// R Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMod(double value) { return BinaryOperation(value, "%", true); } + /// <summary> + /// R Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMod(short value) { return BinaryOperation(value, "%", true); } + /// <summary> + /// R Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMod(long value) { return BinaryOperation(value, "%", true); } + /// <summary> + /// R Mod + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RMod(Column value) { return BinaryOperation(value, "%", true); } + /// <summary> + /// Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Pow(int value) { return BinaryOperation(value, "power"); } + /// <summary> + /// Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Pow(float value) { return BinaryOperation(value, "power"); } + /// <summary> + /// Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Pow(double value) { return BinaryOperation(value, "power"); } + /// <summary> + /// Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Pow(short value) { return BinaryOperation(value, "power"); } + /// <summary> + /// Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Pow(long value) { return BinaryOperation(value, "power"); } + /// <summary> + /// Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Pow(Column value) { return BinaryOperation(value, "power"); } + /// <summary> + /// R Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RPow(int value) { return BinaryOperation(value, "power", true); } + /// <summary> + /// R Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RPow(float value) { return BinaryOperation(value, "power", true); } + /// <summary> + /// R Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RPow(double value) { return BinaryOperation(value, "power", true); } + /// <summary> + /// R Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RPow(short value) { return BinaryOperation(value, "power", true); } + /// <summary> + /// R Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RPow(long value) { return BinaryOperation(value, "power", true); } + /// <summary> + /// R Pow + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column RPow(Column value) { return BinaryOperation(value, "power", true); } + /// <summary> + /// Greater than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >(Column col, int value) { return col.Gt(value); } + /// <summary> + /// Greater than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >(Column col, float value) { return col.Gt(value); } + /// <summary> + /// Greater than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >(Column col, double value) { return col.Gt(value); } + /// <summary> + /// Greater than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >(Column col, short value) { return col.Gt(value); } + /// <summary> + /// Greater than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >(Column col, long value) { return col.Gt(value); } + /// <summary> + /// Greater than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >(Column col, Column value) { return col.Gt(value); } + /// <summary> + /// Less than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <(Column col, int value) { return col.Lt(value); } + /// <summary> + /// Less than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <(Column col, float value) { return col.Lt(value); } + /// <summary> + /// Less than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <(Column col, double value) { return col.Lt(value); } + /// <summary> + /// Less than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <(Column col, short value) { return col.Lt(value); } + /// <summary> + /// Less than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <(Column col, long value) { return col.Lt(value); } + /// <summary> + /// Less than + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <(Column col, Column value) { return col.Lt(value); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <=(Column col, Column value) { return col.Le(value); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <=(Column col, int value) { return col.Le(value); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <=(Column col, float value) { return col.Le(value); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <=(Column col, double value) { return col.Le(value); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <=(Column col, short value) { return col.Le(value); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator <=(Column col, long value) { return col.Le(value); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >=(Column col, Column value) { return col.Ge(value); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >=(Column col, int value) { return col.Ge(value); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >=(Column col, float value) { return col.Ge(value); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >=(Column col, double value) { return col.Ge(value); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >=(Column col, short value) { return col.Ge(value); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="col"></param> + /// <param name="value"></param> + /// <returns></returns> public static Column operator >=(Column col, long value) { return col.Ge(value); } + /// <summary> + /// Greater Than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Gt(int value) { return BinaryOperation(value, ">"); } + /// <summary> + /// Greater Than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Gt(float value) { return BinaryOperation(value, ">"); } + /// <summary> + /// Greater Than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Gt(double value) { return BinaryOperation(value, ">"); } + /// <summary> + /// Greater Than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Gt(short value) { return BinaryOperation(value, ">"); } + /// <summary> + /// Greater Than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Gt(long value) { return BinaryOperation(value, ">"); } + /// <summary> + /// Greater Than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Gt(Column value) { return BinaryOperation(value, ">"); } - + + /// <summary> + /// Less than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Lt(int value) { return BinaryOperation(value, "<"); } + /// <summary> + /// Less than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Lt(float value) { return BinaryOperation(value, "<"); } + /// <summary> + /// Less than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Lt(double value) { return BinaryOperation(value, "<"); } + /// <summary> + /// Less than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Lt(short value) { return BinaryOperation(value, "<"); } + /// <summary> + /// Less than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Lt(long value) { return BinaryOperation(value, "<"); } + /// <summary> + /// Less than + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Lt(Column value) { return BinaryOperation(value, "<"); } - + + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Ge(int value) { return BinaryOperation(value, ">="); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Ge(float value) { return BinaryOperation(value, ">="); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Ge(double value) { return BinaryOperation(value, ">="); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Ge(short value) { return BinaryOperation(value, ">="); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Ge(long value) { return BinaryOperation(value, ">="); } + /// <summary> + /// Greater than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Ge(Column value) { return BinaryOperation(value, ">="); } - + + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Le(int value) { return BinaryOperation(value, "<="); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Le(float value) { return BinaryOperation(value, "<="); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Le(double value) { return BinaryOperation(value, "<="); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Le(short value) { return BinaryOperation(value, "<="); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Le(long value) { return BinaryOperation(value, "<="); } + /// <summary> + /// Less than or equal + /// </summary> + /// <param name="value"></param> + /// <returns></returns> public Column Le(Column value) { return BinaryOperation(value, "<="); @@ -1320,11 +2342,21 @@ private Column BinaryOperation(string value, string functionName, bool reverse = return new Column(expression); } + /// <summary> + /// Over + /// </summary> + /// <param name="window"></param> + /// <returns></returns> public Column Over(WindowSpec window) { return window.ToCol(Expression); } + /// <summary> + /// Cast to the new type + /// </summary> + /// <param name="type"></param> + /// <returns>Column</returns> public Column Cast(SparkDataType type) { return new Column(new Expression @@ -1336,11 +2368,20 @@ public Column Cast(SparkDataType type) }); } + /// <summary> + /// Cast to the name (string, int, etc) + /// </summary> + /// <param name="type"></param> + /// <returns></returns> public Column Cast(string type) { return Cast(SparkDataType.FromString(type)); } + /// <summary> + /// Is the column null? + /// </summary> + /// <returns></returns> public Column IsNull() { var expression = new Expression @@ -1356,6 +2397,10 @@ public Column IsNull() return new Column(expression); } + /// <summary> + /// Is it not null? + /// </summary> + /// <returns></returns> public Column IsNotNull() { var expression = new Expression @@ -1371,6 +2416,11 @@ public Column IsNotNull() return new Column(expression); } + /// <summary> + /// Does it end with the string other? + /// </summary> + /// <param name="other"></param> + /// <returns></returns> public Column EndsWith(string other) { var expression = new Expression @@ -1394,6 +2444,11 @@ public Column EndsWith(string other) return new Column(expression); } + /// <summary> + /// Does it start with the string other? + /// </summary> + /// <param name="other"></param> + /// <returns></returns> public Column StartsWith(string other) { var expression = new Expression @@ -1417,6 +2472,12 @@ public Column StartsWith(string other) return new Column(expression); } + /// <summary> + /// Is it between these two other columns? + /// </summary> + /// <param name="lowerBound"></param> + /// <param name="upperBound"></param> + /// <returns></returns> public Column Between(Column lowerBound, Column upperBound) { return (this >= lowerBound).And(this <= upperBound); @@ -1454,6 +2515,11 @@ public Column Otherwise(Column value) return this; } + /// <summary> + /// Drop these field names + /// </summary> + /// <param name="fieldNames"></param> + /// <returns></returns> public Column DropFields(params string[] fieldNames) { var lastExpression = Expression; @@ -1474,6 +2540,11 @@ public Column DropFields(params string[] fieldNames) return new Column(lastExpression); } + /// <summary> + /// Get a column by field name + /// </summary> + /// <param name="name"></param> + /// <returns></returns> public Column GetField(string name) { var expression = new Expression @@ -1487,6 +2558,11 @@ public Column GetField(string name) return new Column(expression); } + /// <summary> + /// Is it Like other (remember case sensitive) + /// </summary> + /// <param name="other"></param> + /// <returns></returns> public Column Like(string other) { var expression = new Expression @@ -1510,6 +2586,11 @@ public Column Like(string other) return new Column(expression); } + /// <summary> + /// Case-insensitive like + /// </summary> + /// <param name="other"></param> + /// <returns></returns> public Column ILike(string other) { var expression = new Expression @@ -1533,6 +2614,11 @@ public Column ILike(string other) return new Column(expression); } + /// <summary> + /// R Like + /// </summary> + /// <param name="other"></param> + /// <returns></returns> public Column RLike(string other) { var expression = new Expression @@ -1556,6 +2642,12 @@ public Column RLike(string other) return new Column(expression); } + /// <summary> + /// Extract the substring and return a new `Column` + /// </summary> + /// <param name="startPos"></param> + /// <param name="length"></param> + /// <returns>Column</returns> public Column Substr(int startPos, int length) { var expression = new Expression @@ -1587,11 +2679,21 @@ public Column Substr(int startPos, int length) return new Column(expression); } + /// <summary> + /// Is the value in the column in the list of values? + /// </summary> + /// <param name="cols"></param> + /// <returns>Column</returns> public Column IsIn(params object[] cols) { return IsIn(cols.ToList()); } + /// <summary> + /// Is the value in the column in the list of values? + /// </summary> + /// <param name="cols"></param> + /// <returns>Column</returns> public Column IsIn(List<object> cols) { var expression = new Expression diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs index 3d89df6..b5e7acc 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Functions.cs @@ -1548,7 +1548,7 @@ public static Column Randn(Column seed) /// <Summary> /// Round /// Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// </Summary> public static Column Round(string col, Column scale) { @@ -1558,7 +1558,7 @@ public static Column Round(string col, Column scale) /// <Summary> /// Round /// Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// </Summary> public static Column Round(Column col, Column scale) { @@ -1568,7 +1568,7 @@ public static Column Round(Column col, Column scale) /// <Summary> /// Bround /// Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// </Summary> public static Column Bround(string col, Column scale) { @@ -1578,7 +1578,7 @@ public static Column Bround(string col, Column scale) /// <Summary> /// Bround /// Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// </Summary> public static Column Bround(Column col, Column scale) { diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs deleted file mode 100644 index ce70f42..0000000 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/Internal/CreateDataFrameHelper.cs +++ /dev/null @@ -1,20 +0,0 @@ -namespace Spark.Connect.Dotnet.Sql; - -public class CreateDataFrameHelper -{ - /// <summary> - /// Creates a `DataFrame` from a single list that contains one column - /// </summary> - /// <param name="items"></param> - /// <returns></returns> - // public LocalRelation Create(List<object> items, string colName) - // { - // var a = new StructType(new[]{new Field("a", new Int32Type(), false, new List<KeyValuePair<string, string>>())}); - // - // } - // - // public LocalRelation Create(List<List<object>> rows) - // { - // - // } -} \ No newline at end of file diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs index 50bd7e5..8d7b0df 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/ManualFunctions.cs @@ -2024,7 +2024,7 @@ public static Column Struct(params Column[] cols) /// <Summary> /// Round /// Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 or at integral part - /// when `scale` < 0. + /// when `scale` < 0. /// </Summary> public static Column Round(Column col, int scale) { diff --git a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs index c803645..a607583 100644 --- a/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs +++ b/src/Spark.Connect.Dotnet/Spark.Connect.Dotnet/Sql/SparkSession.cs @@ -305,7 +305,7 @@ public DataFrame Sql(string sql) /// ```csharp /// var df = spark.Range(100); /// var dict = new Dictionary - /// <string, object> + /// <string, object> /// (); /// dict["c"] = df["id"]; //could do Col("id") etc /// dict["dataFramePassedIn"] = df; @@ -361,7 +361,7 @@ public DataFrame Sql(string sql, params (string, DataFrame)[] dataFrames) /// <summary> /// Pass in a list of tuples, schema is guessed by the type of the first tuple's child types: /// CreateDataFrame(new List - /// <object, object> + /// <object, object> /// (){ /// ("tupple", 1), ("another", 2) /// }); @@ -399,7 +399,7 @@ public DataFrame CreateDataFrame(IEnumerable<(object, object)> rows, string cola /// <summary> /// Pass in a list of tuples, schema is guessed by the type of the first tuple's child types: /// CreateDataFrame(new List - /// <object, object> + /// <object, object> /// (){ /// ("tupple", 1), ("another", 2) /// }); @@ -435,7 +435,7 @@ public DataFrame CreateDataFrame(IEnumerable<(object, object)> rows) /// <summary> /// Pass in a List of Dictionary - /// <string, type> - the key is used as the name of the field. The schema is guessed at using the first dictionary + /// <string, type> - the key is used as the name of the field. The schema is guessed at using the first dictionary /// </summary> /// <param name="rows"></param> /// <returns></returns> @@ -455,7 +455,7 @@ public DataFrame CreateDataFrame(IEnumerable<Dictionary<string, object>> rows) /// <summary> /// Pass in a List of Dictionary - /// <string, type> - the key is used as the name of the field. The schema is passed in explicitly + /// <string, type> - the key is used as the name of the field. The schema is passed in explicitly /// </summary> /// <param name="rows"></param> /// <param name="schema"></param> @@ -519,16 +519,16 @@ public DataFrame CreateDataFrame(IEnumerable<IEnumerable<object>> data, params s /// <summary> /// Pass in rows and an explicit schema: /// CreateDataFrame(new List - /// <object> + /// <object> /// (){ /// new List - /// <object> + /// <object> /// (){"abc", 123, 100.123}, /// new List - /// <object> + /// <object> /// (){"def", 456, 200.456}, /// new List - /// <object> + /// <object> /// (){"xyz", 999, 999.456}, /// }, new StructType( /// new StructField("col_a", StringType(), true),