External Execution Interface

src/cs/lib/msquic_generated.cs

@@ -234,6 +234,40 @@
         internal fixed ushort ProcessorList[1];
     }
 
+    internal unsafe partial struct QUIC_EXECUTION_CONTEXT_CONFIG
+    {
+        [NativeTypeName("uint32_t")]
+        internal uint IdealProcessor;
+
+        [NativeTypeName("uint32_t")]
+        internal uint PollingIdleTimeoutUs;
+
+        [NativeTypeName("QUIC_EVENTQ *")]
+        internal void** EventQ;
+    }
+
+    internal partial struct QUIC_EXECUTION_CONTEXT
+    {
+    }
+
+    internal unsafe partial struct QUIC_EXECUTION_TABLE
+    {
+        [NativeTypeName("QUIC_EXECUTION_CREATE_FN")]
+        internal delegate* unmanaged[Cdecl]<QUIC_EXECUTION_CONFIG_FLAGS, uint, QUIC_EXECUTION_CONTEXT_CONFIG*, QUIC_EXECUTION_CONTEXT**, int> ExecutionCreate;
+
+        [NativeTypeName("QUIC_EXECUTION_DELETE_FN")]
+        internal delegate* unmanaged[Cdecl]<uint, QUIC_EXECUTION_CONTEXT**, void> ExecutionDelete;
+
+        [NativeTypeName("QUIC_EXECUTION_POLL_FN")]
+        internal delegate* unmanaged[Cdecl]<QUIC_EXECUTION_CONTEXT*, uint> Poll;
+
+        [NativeTypeName("QUIC_EXECUTION_CHECK_CQE_FN")]
+        internal delegate* unmanaged[Cdecl]<_OVERLAPPED_ENTRY*, byte> CheckCqe;
+
+        [NativeTypeName("QUIC_EXECUTION_PROCESS_CQE_FN")]
+        internal delegate* unmanaged[Cdecl]<QUIC_EXECUTION_CONTEXT*, _OVERLAPPED_ENTRY*, uint, uint> ProcessCqe;
+    }
+
     internal unsafe partial struct QUIC_REGISTRATION_CONFIG
     {
         [NativeTypeName("const char *")]
@@ -3345,6 +3379,9 @@
         [NativeTypeName("#define QUIC_PARAM_GLOBAL_STATELESS_RESET_KEY 0x0100000B")]
         internal const uint QUIC_PARAM_GLOBAL_STATELESS_RESET_KEY = 0x0100000B;
 
+        [NativeTypeName("#define QUIC_PARAM_GLOBAL_EXECUTION_TABLE 0x0100000C")]
+        internal const uint QUIC_PARAM_GLOBAL_EXECUTION_TABLE = 0x0100000C;
+
         [NativeTypeName("#define QUIC_PARAM_CONFIGURATION_SETTINGS 0x03000000")]
         internal const uint QUIC_PARAM_CONFIGURATION_SETTINGS = 0x03000000;
 

src/inc/msquic.h

@@ -264,15 +264,15 @@ typedef enum QUIC_DATAGRAM_SEND_STATE {
 #define QUIC_DATAGRAM_SEND_STATE_IS_FINAL(State) \
     ((State) >= QUIC_DATAGRAM_SEND_LOST_DISCARDED)
 
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+
 typedef enum QUIC_EXECUTION_CONFIG_FLAGS {
     QUIC_EXECUTION_CONFIG_FLAG_NONE             = 0x0000,
-#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
     QUIC_EXECUTION_CONFIG_FLAG_QTIP             = 0x0001,
     QUIC_EXECUTION_CONFIG_FLAG_RIO              = 0x0002,
     QUIC_EXECUTION_CONFIG_FLAG_XDP              = 0x0004,
     QUIC_EXECUTION_CONFIG_FLAG_NO_IDEAL_PROC    = 0x0008,
     QUIC_EXECUTION_CONFIG_FLAG_HIGH_PRIORITY    = 0x0010,
-#endif
 } QUIC_EXECUTION_CONFIG_FLAGS;
 
 DEFINE_ENUM_FLAG_OPERATORS(QUIC_EXECUTION_CONFIG_FLAGS)
@@ -293,6 +293,100 @@ typedef struct QUIC_EXECUTION_CONFIG {
 #define QUIC_EXECUTION_CONFIG_MIN_SIZE \
     (uint32_t)FIELD_OFFSET(QUIC_EXECUTION_CONFIG, ProcessorList)
 
+#ifndef _KERNEL_MODE
+
+//
+// Execution Context abstraction, which allows the application layer to
+// completely control execution of all MsQuic work.
+//
+
+typedef struct QUIC_EXECUTION_CONTEXT_CONFIG {
+    uint32_t IdealProcessor;
+    uint32_t PollingIdleTimeoutUs;
+    QUIC_EVENTQ* EventQ;
+} QUIC_EXECUTION_CONTEXT_CONFIG;
+
+typedef struct QUIC_EXECUTION_CONTEXT QUIC_EXECUTION_CONTEXT;
+
+//
+// This is called create the execution contexts.
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+QUIC_STATUS
+(QUIC_API * QUIC_EXECUTION_CREATE_FN)(
+    _In_ QUIC_EXECUTION_CONFIG_FLAGS Flags, // Used for datapath type
+    _In_ uint32_t Count,
+    _In_reads_(Count) QUIC_EXECUTION_CONTEXT_CONFIG* Configs,
+    _Out_writes_(Count) QUIC_EXECUTION_CONTEXT** ExecutionContexts
+    );
+
+//
+// This is called to delete the execution contexts.
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+void
+(QUIC_API * QUIC_EXECUTION_DELETE_FN)(
+    _In_ uint32_t Count,
+    _In_reads_(Count) QUIC_EXECUTION_CONTEXT** ExecutionContexts
+    );
+
+//
+// This is called to allow MsQuic to process any polling work. It returns the
+// number of milliseconds until the next scheduled timer expiration.
+//
+// TODO: Should it return an indication for if we should yield?
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+uint32_t
+(QUIC_API * QUIC_EXECUTION_POLL_FN)(
+    _In_ QUIC_EXECUTION_CONTEXT* ExecutionContext
+    );
+
+//
+// This is used to check if a completion event belongs to MsQuic or not.
+//
+// TODO: This makes certain assumptions for the layout of completion event
+// payload. How do we generalize this?
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+BOOLEAN
+(QUIC_API * QUIC_EXECUTION_CHECK_CQE_FN)(
+    _In_ const QUIC_CQE* Cqe
+    );
+
+//
+// This is called to allow MsQuic to process any completions that have occurred.
+//
+typedef
+_IRQL_requires_max_(PASSIVE_LEVEL)
+uint32_t
+(QUIC_API * QUIC_EXECUTION_PROCESS_CQE_FN)(
+    _In_ QUIC_EXECUTION_CONTEXT* ExecutionContext,
+    _In_reads_(CqeCount) QUIC_CQE* Cqes,
+    _In_ uint32_t CqeCount
+    );
+
+//
+// The table of execution functions.
+//
+typedef struct QUIC_EXECUTION_TABLE {
+
+    QUIC_EXECUTION_CREATE_FN ExecutionCreate;
+    QUIC_EXECUTION_DELETE_FN ExecutionDelete;
+    QUIC_EXECUTION_POLL_FN Poll;
+    QUIC_EXECUTION_CHECK_CQE_FN CheckCqe;
+    QUIC_EXECUTION_PROCESS_CQE_FN ProcessCqe;
+
+} QUIC_EXECUTION_TABLE;
+
+#endif // _KERNEL_MODE
+
+#endif // QUIC_API_ENABLE_PREVIEW_FEATURES
+
 typedef struct QUIC_REGISTRATION_CONFIG { // All fields may be NULL/zero.
     const char* AppName;
     QUIC_EXECUTION_PROFILE ExecutionProfile;
@@ -857,6 +951,10 @@ void
 #endif
 #define QUIC_PARAM_GLOBAL_TLS_PROVIDER                  0x0100000A  // QUIC_TLS_PROVIDER
 #define QUIC_PARAM_GLOBAL_STATELESS_RESET_KEY           0x0100000B  // uint8_t[] - Array size is QUIC_STATELESS_RESET_KEY_LENGTH
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+#define QUIC_PARAM_GLOBAL_EXECUTION_TABLE               0x0100000C  // QUIC_EXECUTION_TABLE
+#endif
+
 //
 // Parameters for Registration.
 //

src/inc/msquic_posix.h

@@ -515,6 +515,26 @@ QuicAddrToString(
     return TRUE;
 }
 
+//
+// Event Queue Abstraction
+//
+
+#if __linux__ // epoll
+
+typedef int QUIC_EVENTQ;
+typedef struct epoll_event QUIC_CQE;
+
+#elif __APPLE__ || __FreeBSD__ // kqueue
+
+typedef int QUIC_EVENTQ;
+typedef struct kevent QUIC_CQE;
+
+#else
+
+#error Unsupported Platform
+
+#endif
+
 #if defined(__cplusplus)
 }
 #endif

src/inc/msquic_winuser.h

@@ -373,4 +373,11 @@ QuicAddrToString(
 
 #endif // WINAPI_FAMILY != WINAPI_FAMILY_GAMES
 
+//
+// Event Queue Abstraction
+//
+
+typedef HANDLE QUIC_EVENTQ;
+typedef OVERLAPPED_ENTRY QUIC_CQE;
+
 #endif // _MSQUIC_WINUSER_

src/tools/execution/CMakeLists.txt

@@ -0,0 +1,5 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+
+add_quic_tool(quicexecution execution_windows.cpp)
+quic_tool_warnings(quicexecution)

src/tools/execution/execution_windows.cpp

@@ -0,0 +1,109 @@
+/*++
+
+    Copyright (c) Microsoft Corporation.
+    Licensed under the MIT License.
+
+Abstract:
+
+    Provides simple client MsQuic example that leverages custom execution.
+
+--*/
+
+#define _CRT_SECURE_NO_WARNINGS 1
+#define QUIC_API_ENABLE_PREVIEW_FEATURES 1
+
+#include "msquic.hpp"
+#include <stdio.h>
+#include <stdlib.h>
+
+const MsQuicApi* MsQuic;
+
+void PrintUsage()
+{
+    printf(
+        "\n"
+        "quicexec is a simple app that can connect to an HTTP/3 server.\n"
+        "\n"
+        "Usage:\n"
+        "\n"
+        "  quicexec <host or ip>\n"
+        );
+}
+
+int
+QUIC_MAIN_EXPORT
+main(
+    _In_ int argc,
+    _In_reads_(argc) _Null_terminated_ char* argv[]
+    )
+{
+    MsQuicApi _MsQuic;
+    if (!_MsQuic.IsValid()) { return 1; }
+    MsQuic = &_MsQuic;
+
+    QUIC_STATUS Status;
+    QUIC_EXECUTION_TABLE MsQuicExec;
+    uint32_t MsQuicExecLength = sizeof(MsQuicExec);
+    if (QUIC_FAILED(
+        Status = MsQuic->GetParam(
+            nullptr,
+            QUIC_PARAM_GLOBAL_EXECUTION_TABLE,
+            &MsQuicExecLength,
+            &MsQuicExec))) {
+        return 1;
+    }
+
+    HANDLE IOCP = CreateIoCompletionPort(INVALID_HANDLE_VALUE, nullptr, 0, 1);
+    QUIC_EXECUTION_CONTEXT_CONFIG ExecConfig = { 0, 0, &IOCP };
+    QUIC_EXECUTION_CONTEXT* ExecContext = nullptr;
+    if (QUIC_FAILED(
+        Status = MsQuicExec.ExecutionCreate(
+            QUIC_EXECUTION_CONFIG_FLAG_NONE,
+            1,
+            &ExecConfig,
+            &ExecContext))) {
+        return 1;
+    }
+
+    do {
+        MsQuicRegistration Registration("quicexec");
+        MsQuicSettings Settings;
+        Settings.SetPeerUnidiStreamCount(3); // required for H3
+        MsQuicConfiguration Configuration(Registration, "h3", Settings, MsQuicCredentialConfig());
+        if (!Configuration.IsValid()) { break; }
+
+        MsQuicConnection Connection(Registration);
+        if (QUIC_FAILED(
+            Status = Connection.Start(Configuration, argv[1], 443))) {
+            break;
+        }
+
+        while (true) {
+            uint32_t WaitTime = MsQuicExec.Poll(ExecContext);
+
+            OVERLAPPED_ENTRY Cqes[8];
+            ULONG CqeCount = 0;
+            if (GetQueuedCompletionStatusEx(IOCP, Cqes, ARRAYSIZE(Cqes), &CqeCount, WaitTime, FALSE)) {
+                for (ULONG i = 0; i < CqeCount; ++i) {
+                    if (MsQuicExec.CheckCqe(Cqes+i)) {
+                        MsQuicExec.ProcessCqe(ExecContext, Cqes+i, 1);
+                    } else {
+                        // We should handle our own completions here.
+                    }
+                }
+            }
+
+            if (Connection.HandshakeComplete) {
+                Connection.Shutdown(0);
+            }
+
+            // TODO - Stop once the connection is shutdown complete
+        }
+
+    } while (false);
+
+    MsQuicExec.ExecutionDelete(1, &ExecContext);
+    CloseHandle(IOCP);
+
+    return 0;
+}