diff --git a/.github/workflows/release-worker.yml b/.github/workflows/release-worker.yml index 71e6565a1..86aff4128 100644 --- a/.github/workflows/release-worker.yml +++ b/.github/workflows/release-worker.yml @@ -32,7 +32,7 @@ jobs: id: login-ecr with: registry-type: public - mask-password: 'true' + mask-password: "true" - name: Set version id: set-version @@ -49,3 +49,6 @@ jobs: ${{ steps.login-ecr.outputs.registry }}/n4e0e1y0/beta9-worker:latest target: final platforms: linux/amd64 + build-args: | + CEDANA_TOKEN=${{ secrets.CEDANA_TOKEN }} + CEDANA_BASE_URL=${{ secrets.CEDANA_BASE_URL }} diff --git a/.stignore b/.stignore index 662b68653..827da1d92 100644 --- a/.stignore +++ b/.stignore @@ -25,3 +25,8 @@ venv test manifests tmp + + +deploy +bin/worker +bin/gateway \ No newline at end of file diff --git a/bin/setup.sh b/bin/setup.sh old mode 100644 new mode 100755 diff --git a/docker/Dockerfile.runner b/docker/Dockerfile.runner index 1e155182e..543a81a03 100644 --- a/docker/Dockerfile.runner +++ b/docker/Dockerfile.runner @@ -1,12 +1,12 @@ # syntax=docker/dockerfile:1.6 -FROM ubuntu:20.04 as base +FROM ubuntu:22.04 as base ENV DEBIAN_FRONTEND=noninteractive RUN < /etc/apt/sources.list.d/criu.list && \ curl -fsSL https://nvidia.github.io/nvidia-container-runtime/gpgkey | apt-key add - && \ - curl -s -L https://nvidia.github.io/nvidia-docker/ubuntu20.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list \ - curl -fsSL https://nvidia.github.io/nvidia-container-runtime/ubuntu20.04/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list && \ + curl -s -L https://nvidia.github.io/nvidia-docker/ubuntu22.04/nvidia-docker.list | sudo tee /etc/apt/sources.list.d/nvidia-docker.list \ + curl -fsSL https://nvidia.github.io/nvidia-container-runtime/ubuntu22.04/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list && \ apt-get update && \ apt-get install psmisc RUN curl -L https://beam-runner-python-deps.s3.amazonaws.com/juicefs -o /usr/local/bin/juicefs && chmod +x /usr/local/bin/juicefs RUN curl -fsSL https://tailscale.com/install.sh | sh -RUN apt-get install -y --no-install-recommends criu nvidia-container-toolkit-base nvidia-container-toolkit +RUN apt-get install -y --no-install-recommends nvidia-container-toolkit-base nvidia-container-toolkit RUN apt-get update && apt-get install -y fuse3 libfuse2 libfuse3-dev libfuse-dev bash-completion +# XXX: Remove once cedana starts shipping with a compatible binary +RUN < 0 { + gpuCount := stubConfig.Runtime.GpuCount + if stubConfig.RequiresGPU() && gpuCount == 0 { gpuCount = 1 } diff --git a/pkg/abstractions/endpoint/instance.go b/pkg/abstractions/endpoint/instance.go index d503e4be0..ffb4c75ec 100644 --- a/pkg/abstractions/endpoint/instance.go +++ b/pkg/abstractions/endpoint/instance.go @@ -68,28 +68,38 @@ func (i *endpointInstance) startContainers(containersToRun int) error { gpuRequest = append(gpuRequest, i.StubConfig.Runtime.Gpu.String()) } - gpuCount := 0 - if len(gpuRequest) > 0 { + gpuCount := i.StubConfig.Runtime.GpuCount + if i.StubConfig.RequiresGPU() && gpuCount == 0 { gpuCount = 1 } + checkpointEnabled := i.StubConfig.CheckpointEnabled + if i.Stub.Type.IsServe() { + checkpointEnabled = false + } + + if gpuCount > 1 { + checkpointEnabled = false + } + for c := 0; c < containersToRun; c++ { containerId := i.genContainerId() runRequest := &types.ContainerRequest{ - ContainerId: containerId, - Env: env, - Cpu: i.StubConfig.Runtime.Cpu, - Memory: i.StubConfig.Runtime.Memory, - GpuRequest: gpuRequest, - GpuCount: uint32(gpuCount), - ImageId: i.StubConfig.Runtime.ImageId, - StubId: i.Stub.ExternalId, - WorkspaceId: i.Workspace.ExternalId, - Workspace: *i.Workspace, - EntryPoint: i.EntryPoint, - Mounts: mounts, - Stub: *i.Stub, + ContainerId: containerId, + Env: env, + Cpu: i.StubConfig.Runtime.Cpu, + Memory: i.StubConfig.Runtime.Memory, + GpuRequest: gpuRequest, + GpuCount: uint32(gpuCount), + ImageId: i.StubConfig.Runtime.ImageId, + StubId: i.Stub.ExternalId, + WorkspaceId: i.Workspace.ExternalId, + Workspace: *i.Workspace, + EntryPoint: i.EntryPoint, + Mounts: mounts, + Stub: *i.Stub, + CheckpointEnabled: checkpointEnabled, } // Set initial keepwarm to prevent rapid spin-up/spin-down of containers diff --git a/pkg/abstractions/endpoint/task.go b/pkg/abstractions/endpoint/task.go index ff0742b57..a3d0f7174 100644 --- a/pkg/abstractions/endpoint/task.go +++ b/pkg/abstractions/endpoint/task.go @@ -90,3 +90,7 @@ func (t *EndpointTask) Metadata() types.TaskMetadata { TaskId: t.msg.TaskId, } } + +func (t *EndpointTask) Message() *types.TaskMessage { + return t.msg +} diff --git a/pkg/abstractions/experimental/bot/http.go b/pkg/abstractions/experimental/bot/http.go index a9aef0f6c..2b4a0cfa8 100644 --- a/pkg/abstractions/experimental/bot/http.go +++ b/pkg/abstractions/experimental/bot/http.go @@ -138,6 +138,20 @@ func (g *botGroup) BotOpenSession(ctx echo.Context) error { if err != nil { return err } + + if instance.botConfig.WelcomeMessage != "" { + err = instance.botStateManager.pushEvent(instance.workspace.Name, instance.stub.ExternalId, sessionId, &BotEvent{ + Type: BotEventTypeAgentMessage, + Value: instance.botConfig.WelcomeMessage, + Metadata: map[string]string{ + string(MetadataSessionId): sessionId, + }, + }) + + if err != nil { + return err + } + } } stubType = types.StubType(instance.stub.Type) diff --git a/pkg/abstractions/experimental/bot/state.go b/pkg/abstractions/experimental/bot/state.go index b1197768b..52557af55 100644 --- a/pkg/abstractions/experimental/bot/state.go +++ b/pkg/abstractions/experimental/bot/state.go @@ -264,6 +264,12 @@ func (m *botStateManager) pushUserEvent(workspaceName, stubId, sessionId string, return err } + historyKey := Keys.botEventHistory(workspaceName, stubId, sessionId) + err = m.rdb.RPush(context.TODO(), historyKey, jsonData).Err() + if err != nil { + return err + } + messageKey := Keys.botInputBuffer(workspaceName, stubId, sessionId) return m.rdb.RPush(context.TODO(), messageKey, jsonData).Err() } @@ -292,12 +298,14 @@ func (m *botStateManager) pushEvent(workspaceName, stubId, sessionId string, eve messageKey := Keys.botEventBuffer(workspaceName, stubId, sessionId) historyKey := Keys.botEventHistory(workspaceName, stubId, sessionId) - err = m.rdb.RPush(context.TODO(), messageKey, jsonData).Err() - if err != nil { - return err + if event.Type != BotEventTypeNetworkState { + err = m.rdb.RPush(context.TODO(), historyKey, jsonData).Err() + if err != nil { + return err + } } - return m.rdb.RPush(context.TODO(), historyKey, jsonData).Err() + return m.rdb.RPush(context.TODO(), messageKey, jsonData).Err() } const eventPairTtlS = 120 * time.Second diff --git a/pkg/abstractions/experimental/bot/task.go b/pkg/abstractions/experimental/bot/task.go index 5ccd9dbd9..653b315c5 100644 --- a/pkg/abstractions/experimental/bot/task.go +++ b/pkg/abstractions/experimental/bot/task.go @@ -98,3 +98,7 @@ func (t *BotTask) Metadata() types.TaskMetadata { WorkspaceName: t.msg.WorkspaceName, } } + +func (t *BotTask) Message() *types.TaskMessage { + return t.msg +} diff --git a/pkg/abstractions/experimental/bot/types.go b/pkg/abstractions/experimental/bot/types.go index 958bec390..f28df808a 100644 --- a/pkg/abstractions/experimental/bot/types.go +++ b/pkg/abstractions/experimental/bot/types.go @@ -153,11 +153,12 @@ type MarkerField struct { // BotConfig holds the overall config for the bot type BotConfig struct { - Model string `json:"model" redis:"model"` - Locations map[string]BotLocationConfig `json:"locations" redis:"locations"` - Transitions map[string]BotTransitionConfig `json:"transitions" redis:"transitions"` - ApiKey string `json:"api_key" redis:"api_key"` - Authorized bool `json:"authorized" redis:"authorized"` + Model string `json:"model" redis:"model"` + Locations map[string]BotLocationConfig `json:"locations" redis:"locations"` + Transitions map[string]BotTransitionConfig `json:"transitions" redis:"transitions"` + ApiKey string `json:"api_key" redis:"api_key"` + Authorized bool `json:"authorized" redis:"authorized"` + WelcomeMessage string `json:"welcome_message" redis:"welcome_message"` } func (b *BotConfig) FormatLocations() string { diff --git a/pkg/abstractions/function/task.go b/pkg/abstractions/function/task.go index b409c7245..896edf52c 100644 --- a/pkg/abstractions/function/task.go +++ b/pkg/abstractions/function/task.go @@ -145,8 +145,8 @@ func (t *FunctionTask) run(ctx context.Context, stub *types.StubWithRelated) err gpuRequest = append(gpuRequest, stubConfig.Runtime.Gpu.String()) } - gpuCount := 0 - if len(gpuRequest) > 0 { + gpuCount := stubConfig.Runtime.GpuCount + if stubConfig.RequiresGPU() && gpuCount == 0 { gpuCount = 1 } @@ -212,3 +212,7 @@ func (t *FunctionTask) Metadata() types.TaskMetadata { ContainerId: t.containerId, } } + +func (t *FunctionTask) Message() *types.TaskMessage { + return t.msg +} diff --git a/pkg/abstractions/image/build.go b/pkg/abstractions/image/build.go index adb30cdc9..a706d01b9 100644 --- a/pkg/abstractions/image/build.go +++ b/pkg/abstractions/image/build.go @@ -360,6 +360,11 @@ func (b *Builder) genContainerId() string { } func (b *Builder) extractPackageName(pkg string) string { + // For now we let this go through and let the pip install command fail if the package is not found + if len(pkg) == 0 { + return "" + } + // Handle Git URLs if strings.HasPrefix(pkg, "git+") || strings.HasPrefix(pkg, "-e git+") { if eggTag := strings.Split(pkg, "#egg="); len(eggTag) > 1 { @@ -617,12 +622,27 @@ func parseBuildSteps(buildSteps []BuildStep, pythonVersion string) []string { commands = append(commands, step.Command) } + flagCmd := containsFlag(step.Command) + + // Flush any pending pip or mamba groups + if pipStart != -1 && (step.Type != pipCommandType || flagCmd) { + pipStart, pipGroup = flushPipCommand(commands, pipStart, pipGroup, pythonVersion) + } + + if mambaStart != -1 && (step.Type != micromambaCommandType || flagCmd) { + mambaStart, mambaGroup = flushMambaCommand(commands, mambaStart, mambaGroup) + } + if step.Type == pipCommandType { if pipStart == -1 { pipStart = len(commands) commands = append(commands, "") } pipGroup = append(pipGroup, step.Command) + + if flagCmd { + pipStart, pipGroup = flushPipCommand(commands, pipStart, pipGroup, pythonVersion) + } } if step.Type == micromambaCommandType { @@ -631,19 +651,10 @@ func parseBuildSteps(buildSteps []BuildStep, pythonVersion string) []string { commands = append(commands, "") } mambaGroup = append(mambaGroup, step.Command) - } - - // Flush any pending pip or mamba groups - if pipStart != -1 && step.Type != pipCommandType { - commands[pipStart] = generatePipInstallCommand(pipGroup, pythonVersion) - pipStart = -1 - pipGroup = nil - } - if mambaStart != -1 && step.Type != micromambaCommandType { - commands[mambaStart] = generateMicromambaInstallCommand(mambaGroup) - mambaStart = -1 - mambaGroup = nil + if flagCmd { + mambaStart, mambaGroup = flushMambaCommand(commands, mambaStart, mambaGroup) + } } } @@ -657,3 +668,37 @@ func parseBuildSteps(buildSteps []BuildStep, pythonVersion string) []string { return commands } + +func flushMambaCommand(commands []string, mambaStart int, mambaGroup []string) (int, []string) { + commands[mambaStart] = generateMicromambaInstallCommand(mambaGroup) + return -1, nil +} + +func flushPipCommand(commands []string, pipStart int, pipGroup []string, pythonVersion string) (int, []string) { + commands[pipStart] = generatePipInstallCommand(pipGroup, pythonVersion) + return -1, nil +} + +func containsFlag(s string) bool { + flags := []string{ + "--no-deps", + "--only-binary", + "--no-binary", + "--prefer-binary", + "--require-hashes", + "--pre", + "--ignore-requires-python", + "--no-pin", + "--force-reinstall", + "--freeze-installed", + "--update-deps", + "--no-update-deps", + } + + for _, flag := range flags { + if strings.Contains(s, flag) { + return true + } + } + return false +} diff --git a/pkg/abstractions/image/build_test.go b/pkg/abstractions/image/build_test.go index fc42b6b0f..f2fddad38 100644 --- a/pkg/abstractions/image/build_test.go +++ b/pkg/abstractions/image/build_test.go @@ -217,6 +217,43 @@ func TestParseBuildSteps(t *testing.T) { }, want: []string{"echo 'hello'", "micromamba3.10 -m pip install --root-user-action=ignore \"numpy\" \"pandas\"", "micromamba install -y -n beta9 \"torch\" \"vllm\"", "apt install -y ffmpeg", "micromamba install -y -n beta9 \"ffmpeg\""}, }, + { + steps: []BuildStep{ + {Type: micromambaCommandType, Command: "torch"}, + {Type: pipCommandType, Command: "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"}, + {Type: pipCommandType, Command: "--no-deps trl peft accelerate bitsandbytes"}, + }, + want: []string{ + "micromamba install -y -n beta9 \"torch\"", + "micromamba3.10 -m pip install --root-user-action=ignore \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"", + "micromamba3.10 -m pip install --root-user-action=ignore --no-deps trl peft accelerate bitsandbytes", + }, + }, + { + steps: []BuildStep{ + {Type: micromambaCommandType, Command: "torch"}, + {Type: pipCommandType, Command: "--no-deps trl peft accelerate bitsandbytes"}, + {Type: pipCommandType, Command: "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"}, + }, + want: []string{ + "micromamba install -y -n beta9 \"torch\"", + "micromamba3.10 -m pip install --root-user-action=ignore --no-deps trl peft accelerate bitsandbytes", + "micromamba3.10 -m pip install --root-user-action=ignore \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"", + }, + }, + { + steps: []BuildStep{ + {Type: micromambaCommandType, Command: "pytorch-cuda=12.1"}, + {Type: micromambaCommandType, Command: "-c pytorch"}, + {Type: pipCommandType, Command: "--no-deps trl peft accelerate bitsandbytes"}, + {Type: pipCommandType, Command: "unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git"}, + }, + want: []string{ + "micromamba install -y -n beta9 -c pytorch \"pytorch-cuda=12.1\"", + "micromamba3.10 -m pip install --root-user-action=ignore --no-deps trl peft accelerate bitsandbytes", + "micromamba3.10 -m pip install --root-user-action=ignore \"unsloth[colab-new] @ git+https://github.com/unslothai/unsloth.git\"", + }, + }, } for _, tc := range testCases { diff --git a/pkg/abstractions/taskqueue/instance.go b/pkg/abstractions/taskqueue/instance.go index 36d0f0b7e..c2f2342de 100644 --- a/pkg/abstractions/taskqueue/instance.go +++ b/pkg/abstractions/taskqueue/instance.go @@ -65,26 +65,36 @@ func (i *taskQueueInstance) startContainers(containersToRun int) error { gpuRequest = append(gpuRequest, i.StubConfig.Runtime.Gpu.String()) } - gpuCount := 0 - if len(gpuRequest) > 0 { + gpuCount := i.StubConfig.Runtime.GpuCount + if i.StubConfig.RequiresGPU() && gpuCount == 0 { gpuCount = 1 } + checkpointEnabled := i.StubConfig.CheckpointEnabled + if i.Stub.Type.IsServe() { + checkpointEnabled = false + } + + if gpuCount > 1 { + checkpointEnabled = false + } + for c := 0; c < containersToRun; c++ { runRequest := &types.ContainerRequest{ - ContainerId: i.genContainerId(), - Env: env, - Cpu: i.StubConfig.Runtime.Cpu, - Memory: i.StubConfig.Runtime.Memory, - GpuRequest: gpuRequest, - GpuCount: uint32(gpuCount), - ImageId: i.StubConfig.Runtime.ImageId, - StubId: i.Stub.ExternalId, - WorkspaceId: i.Workspace.ExternalId, - Workspace: *i.Workspace, - EntryPoint: i.EntryPoint, - Mounts: mounts, - Stub: *i.Stub, + ContainerId: i.genContainerId(), + Env: env, + Cpu: i.StubConfig.Runtime.Cpu, + Memory: i.StubConfig.Runtime.Memory, + GpuRequest: gpuRequest, + GpuCount: uint32(gpuCount), + ImageId: i.StubConfig.Runtime.ImageId, + StubId: i.Stub.ExternalId, + WorkspaceId: i.Workspace.ExternalId, + Workspace: *i.Workspace, + EntryPoint: i.EntryPoint, + Mounts: mounts, + Stub: *i.Stub, + CheckpointEnabled: checkpointEnabled, } err := i.Scheduler.Run(runRequest) diff --git a/pkg/abstractions/taskqueue/keys.go b/pkg/abstractions/taskqueue/keys.go new file mode 100644 index 000000000..eaaf8c8c6 --- /dev/null +++ b/pkg/abstractions/taskqueue/keys.go @@ -0,0 +1,56 @@ +package taskqueue + +import "fmt" + +// Redis keys +var ( + taskQueueList string = "taskqueue:%s:%s" + taskQueueServeLock string = "taskqueue:%s:%s:serve_lock" + taskQueueInstanceLock string = "taskqueue:%s:%s:instance_lock" + taskQueueTaskDuration string = "taskqueue:%s:%s:task_duration" + taskQueueAverageTaskDuration string = "taskqueue:%s:%s:avg_task_duration" + taskQueueTaskHeartbeat string = "taskqueue:%s:%s:task:heartbeat:%s" + taskQueueProcessingLock string = "taskqueue:%s:%s:processing_lock:%s" + taskQueueKeepWarmLock string = "taskqueue:%s:%s:keep_warm_lock:%s" + taskQueueTaskRunningLock string = "taskqueue:%s:%s:task_running:%s:%s" +) + +var Keys = &keys{} + +type keys struct{} + +func (k *keys) taskQueueInstanceLock(workspaceName, stubId string) string { + return fmt.Sprintf(taskQueueInstanceLock, workspaceName, stubId) +} + +func (k *keys) taskQueueServeLock(workspaceName, stubId string) string { + return fmt.Sprintf(taskQueueServeLock, workspaceName, stubId) +} + +func (k *keys) taskQueueList(workspaceName, stubId string) string { + return fmt.Sprintf(taskQueueList, workspaceName, stubId) +} + +func (k *keys) taskQueueTaskHeartbeat(workspaceName, stubId, taskId string) string { + return fmt.Sprintf(taskQueueTaskHeartbeat, workspaceName, stubId, taskId) +} + +func (k *keys) taskQueueTaskDuration(workspaceName, stubId string) string { + return fmt.Sprintf(taskQueueTaskDuration, workspaceName, stubId) +} + +func (k *keys) taskQueueAverageTaskDuration(workspaceName, stubId string) string { + return fmt.Sprintf(taskQueueAverageTaskDuration, workspaceName, stubId) +} + +func (k *keys) taskQueueTaskRunningLock(workspaceName, stubId, containerId, taskId string) string { + return fmt.Sprintf(taskQueueTaskRunningLock, workspaceName, stubId, containerId, taskId) +} + +func (k *keys) taskQueueProcessingLock(workspaceName, stubId, containerId string) string { + return fmt.Sprintf(taskQueueProcessingLock, workspaceName, stubId, containerId) +} + +func (k *keys) taskQueueKeepWarmLock(workspaceName, stubId, containerId string) string { + return fmt.Sprintf(taskQueueKeepWarmLock, workspaceName, stubId, containerId) +} diff --git a/pkg/abstractions/taskqueue/task.go b/pkg/abstractions/taskqueue/task.go index 9beb37d51..86b9e559b 100644 --- a/pkg/abstractions/taskqueue/task.go +++ b/pkg/abstractions/taskqueue/task.go @@ -94,3 +94,7 @@ func (t *TaskQueueTask) Metadata() types.TaskMetadata { WorkspaceName: t.msg.WorkspaceName, } } + +func (t *TaskQueueTask) Message() *types.TaskMessage { + return t.msg +} diff --git a/pkg/abstractions/taskqueue/taskqueue.go b/pkg/abstractions/taskqueue/taskqueue.go index 654df2a32..6b5148efa 100644 --- a/pkg/abstractions/taskqueue/taskqueue.go +++ b/pkg/abstractions/taskqueue/taskqueue.go @@ -345,6 +345,10 @@ func (tq *RedisTaskQueue) TaskQueueComplete(ctx context.Context, in *pb.TaskQueu task.EndedAt = sql.NullTime{Time: time.Now(), Valid: true} task.Status = types.TaskStatus(in.TaskStatus) + if task.Status == types.TaskStatusRetry { + return tq.retryTask(ctx, authInfo, in), nil + } + err = tq.taskDispatcher.Complete(ctx, authInfo.Workspace.Name, in.StubId, in.TaskId) if err != nil { return &pb.TaskQueueCompleteResponse{ @@ -453,6 +457,7 @@ func (tq *RedisTaskQueue) TaskQueueMonitor(req *pb.TaskQueueMonitorRequest, stre for { select { case <-stream.Context().Done(): + tq.rdb.Del(context.Background(), Keys.taskQueueTaskRunningLock(authInfo.Workspace.Name, req.StubId, req.ContainerId, task.ExternalId)) return nil case <-cancelFlag: @@ -484,7 +489,7 @@ func (tq *RedisTaskQueue) TaskQueueMonitor(req *pb.TaskQueueMonitorRequest, stre return err } - err = tq.rdb.SetEx(ctx, Keys.taskQueueTaskRunningLock(authInfo.Workspace.Name, req.StubId, req.ContainerId, task.ExternalId), 1, time.Duration(1)*time.Second).Err() + err = tq.rdb.SetEx(ctx, Keys.taskQueueTaskRunningLock(authInfo.Workspace.Name, req.StubId, req.ContainerId, task.ExternalId), 1, time.Duration(5)*time.Second).Err() if err != nil { return err } @@ -603,55 +608,28 @@ func (tq *RedisTaskQueue) getOrCreateQueueInstance(stubId string, options ...fun return instance, nil } -// Redis keys -var ( - taskQueueList string = "taskqueue:%s:%s" - taskQueueServeLock string = "taskqueue:%s:%s:serve_lock" - taskQueueInstanceLock string = "taskqueue:%s:%s:instance_lock" - taskQueueTaskDuration string = "taskqueue:%s:%s:task_duration" - taskQueueAverageTaskDuration string = "taskqueue:%s:%s:avg_task_duration" - taskQueueTaskHeartbeat string = "taskqueue:%s:%s:task:heartbeat:%s" - taskQueueProcessingLock string = "taskqueue:%s:%s:processing_lock:%s" - taskQueueKeepWarmLock string = "taskqueue:%s:%s:keep_warm_lock:%s" - taskQueueTaskRunningLock string = "taskqueue:%s:%s:task_running:%s:%s" -) - -var Keys = &keys{} - -type keys struct{} - -func (k *keys) taskQueueInstanceLock(workspaceName, stubId string) string { - return fmt.Sprintf(taskQueueInstanceLock, workspaceName, stubId) -} - -func (k *keys) taskQueueServeLock(workspaceName, stubId string) string { - return fmt.Sprintf(taskQueueServeLock, workspaceName, stubId) -} - -func (k *keys) taskQueueList(workspaceName, stubId string) string { - return fmt.Sprintf(taskQueueList, workspaceName, stubId) -} - -func (k *keys) taskQueueTaskHeartbeat(workspaceName, stubId, taskId string) string { - return fmt.Sprintf(taskQueueTaskHeartbeat, workspaceName, stubId, taskId) -} - -func (k *keys) taskQueueTaskDuration(workspaceName, stubId string) string { - return fmt.Sprintf(taskQueueTaskDuration, workspaceName, stubId) -} - -func (k *keys) taskQueueAverageTaskDuration(workspaceName, stubId string) string { - return fmt.Sprintf(taskQueueAverageTaskDuration, workspaceName, stubId) -} +func (tq *RedisTaskQueue) retryTask(ctx context.Context, authInfo *auth.AuthInfo, in *pb.TaskQueueCompleteRequest) *pb.TaskQueueCompleteResponse { + task, err := tq.taskDispatcher.Retrieve(ctx, authInfo.Workspace.Name, in.StubId, in.TaskId) + if err != nil { + return &pb.TaskQueueCompleteResponse{ + Ok: false, + } + } -func (k *keys) taskQueueTaskRunningLock(workspaceName, stubId, containerId, taskId string) string { - return fmt.Sprintf(taskQueueTaskRunningLock, workspaceName, stubId, containerId, taskId) -} + msg := "" + if task.Message().Retries >= task.Message().Policy.MaxRetries { + msg = fmt.Sprintf("Exceeded retry limit of %d for task <%s>", task.Message().Policy.MaxRetries, task.Message().TaskId) + } -func (k *keys) taskQueueProcessingLock(workspaceName, stubId, containerId string) string { - return fmt.Sprintf(taskQueueProcessingLock, workspaceName, stubId, containerId) -} + err = tq.taskDispatcher.RetryTask(ctx, task) + if err != nil { + return &pb.TaskQueueCompleteResponse{ + Ok: false, + } + } -func (k *keys) taskQueueKeepWarmLock(workspaceName, stubId, containerId string) string { - return fmt.Sprintf(taskQueueKeepWarmLock, workspaceName, stubId, containerId) + return &pb.TaskQueueCompleteResponse{ + Ok: true, + Message: msg, + } } diff --git a/pkg/abstractions/taskqueue/taskqueue.proto b/pkg/abstractions/taskqueue/taskqueue.proto index db6f6b129..d8b1b270f 100644 --- a/pkg/abstractions/taskqueue/taskqueue.proto +++ b/pkg/abstractions/taskqueue/taskqueue.proto @@ -58,7 +58,10 @@ message TaskQueueCompleteRequest { float keep_warm_seconds = 7; } -message TaskQueueCompleteResponse { bool ok = 1; } +message TaskQueueCompleteResponse { + bool ok = 1; + string message = 2; +} message TaskQueueMonitorRequest { string task_id = 1; diff --git a/pkg/common/config.default.yaml b/pkg/common/config.default.yaml index a4fde6c87..cd2671517 100644 --- a/pkg/common/config.default.yaml +++ b/pkg/common/config.default.yaml @@ -12,7 +12,7 @@ database: redis: mode: single addrs: - - redis-master:6379 + - redis-master:6379 password: enableTLS: false insecureSkipVerify: false @@ -50,6 +50,7 @@ gateway: stubLimits: memory: 32768 maxReplicas: 10 + maxGpuCount: 2 imageService: localCacheEnabled: true registryStore: local @@ -107,6 +108,7 @@ worker: priority: 1 jobSpec: nodeSelector: {} + criuEnabled: false poolSizing: defaultWorkerCpu: 1000m defaultWorkerGpuType: "" @@ -129,17 +131,19 @@ worker: sharedMemoryLimitPct: 100% # example gpu worker pool # nvidia: - # gpuType: A40 + # mode: local + # gpuType: any # runtime: nvidia # jobSpec: # nodeSelector: {} # poolSizing: - # defaultWorkerCpu: 1000m + # defaultWorkerCpu: # defaultWorkerGpuType: "" - # defaultWorkerMemory: 1Gi + # defaultWorkerMemory: # minFreeCpu: # minFreeGpu: # minFreeMemory: + # sharedMemoryLimitPct: 100% # global pool attributes useHostResolvConf: true hostNetwork: false @@ -159,6 +163,24 @@ worker: terminationGracePeriod: 30 addWorkerTimeout: 10m blobCacheEnabled: false + criu: + storage: + mountPath: /data/checkpoints + mode: local + objectStoreConfig: + bucketName: beta9-checkpoints + accessKey: test + secretKey: test + endpointURL: http://localstack:4566 + cedana: # defaults + client: + leaveRunning: true + sharedStorage: + dumpStorageDir: /tmp/dump + connection: + cedanaUrl: + cedanaAuthToken: + providers: ec2: accessKey: @@ -177,18 +199,18 @@ tailscale: proxy: httpPort: 1989 services: - - name: redis - localPort: 6379 - destination: redis-master:6379 - - name: juicefs-redis - localPort: 6380 - destination: juicefs-redis-master:6379 - - name: gateway-http - localPort: 1994 - destination: beta9-gateway:1994 - - name: gateway-grpc - localPort: 1993 - destination: beta9-gateway:1993 + - name: redis + localPort: 6379 + destination: redis-master:6379 + - name: juicefs-redis + localPort: 6380 + destination: juicefs-redis-master:6379 + - name: gateway-http + localPort: 1994 + destination: beta9-gateway:1994 + - name: gateway-grpc + localPort: 1993 + destination: beta9-gateway:1993 monitoring: containerMetricsInterval: 3s metricsCollector: prometheus diff --git a/pkg/common/keys.go b/pkg/common/keys.go index 1b72d79c6..28cac5151 100644 --- a/pkg/common/keys.go +++ b/pkg/common/keys.go @@ -20,6 +20,7 @@ var ( schedulerWorkerAddress string = "scheduler:container:worker_addr:%s" schedulerContainerLock string = "scheduler:container:lock:%s" schedulerContainerExitCode string = "scheduler:container:exit_code:%s" + schedulerCheckpointState string = "scheduler:checkpoint_state:%s:%s" ) var ( @@ -141,6 +142,10 @@ func (rk *redisKeys) SchedulerContainerExitCode(containerId string) string { return fmt.Sprintf(schedulerContainerExitCode, containerId) } +func (rk *redisKeys) SchedulerCheckpointState(workspaceName, checkpointId string) string { + return fmt.Sprintf(schedulerCheckpointState, workspaceName, checkpointId) +} + // Gateway keys func (rk *redisKeys) GatewayPrefix() string { return gatewayPrefix diff --git a/pkg/gateway/gateway.proto b/pkg/gateway/gateway.proto index d2a656657..b86664261 100644 --- a/pkg/gateway/gateway.proto +++ b/pkg/gateway/gateway.proto @@ -257,6 +257,8 @@ message GetOrCreateStubRequest { TaskPolicy task_policy = 23; uint32 concurrent_requests = 24; string extra = 25; + bool checkpoint_enabled = 26; + uint32 gpu_count = 27; } message GetOrCreateStubResponse { @@ -482,27 +484,21 @@ message ListWorkersResponse { repeated Worker workers = 3; } -message CordonWorkerRequest { - string worker_id = 1; -} +message CordonWorkerRequest { string worker_id = 1; } message CordonWorkerResponse { bool ok = 1; string err_msg = 2; } -message UncordonWorkerRequest { - string worker_id = 1; -} +message UncordonWorkerRequest { string worker_id = 1; } message UncordonWorkerResponse { bool ok = 1; string err_msg = 2; } -message DrainWorkerRequest { - string worker_id = 1; -} +message DrainWorkerRequest { string worker_id = 1; } message DrainWorkerResponse { bool ok = 1; diff --git a/pkg/gateway/services/machine.go b/pkg/gateway/services/machine.go index 3f82abe05..e48696ff9 100644 --- a/pkg/gateway/services/machine.go +++ b/pkg/gateway/services/machine.go @@ -166,6 +166,13 @@ func (gws *GatewayService) CreateMachine(ctx context.Context, in *pb.CreateMachi }, nil } + if pool.Provider == nil { + return &pb.CreateMachineResponse{ + Ok: false, + ErrMsg: "This pool does not currently support machine creation", + }, nil + } + machineId := providers.MachineId() err = gws.providerRepo.AddMachine(string(*pool.Provider), in.PoolName, machineId, &types.ProviderMachineState{ PoolName: in.PoolName, diff --git a/pkg/gateway/services/stub.go b/pkg/gateway/services/stub.go index 6bd3d2953..3fa6bc2b7 100644 --- a/pkg/gateway/services/stub.go +++ b/pkg/gateway/services/stub.go @@ -30,37 +30,6 @@ func (gws *GatewayService) GetOrCreateStub(ctx context.Context, in *pb.GetOrCrea gpus := types.GPUTypesFromString(in.Gpu) - if len(gpus) > 0 { - concurrencyLimit, err := gws.backendRepo.GetConcurrencyLimitByWorkspaceId(ctx, authInfo.Workspace.ExternalId) - if err != nil && concurrencyLimit != nil && concurrencyLimit.GPULimit <= 0 { - return &pb.GetOrCreateStubResponse{ - Ok: false, - ErrMsg: "GPU concurrency limit is 0.", - }, nil - } - - gpuCounts, err := gws.providerRepo.GetGPUCounts(gws.appConfig.Worker.Pools) - if err != nil { - return &pb.GetOrCreateStubResponse{ - Ok: false, - ErrMsg: "Failed to get GPU counts.", - }, nil - } - - // T4s are currently in a different pool than other GPUs and won't show up in gpu counts - lowGpus := []string{} - - for _, gpu := range gpus { - if gpuCounts[gpu.String()] <= 1 && gpu.String() != types.GPU_T4.String() { - lowGpus = append(lowGpus, gpu.String()) - } - } - - if len(lowGpus) > 0 { - warning = fmt.Sprintf("GPU capacity for %s is currently low.", strings.Join(lowGpus, ", ")) - } - } - autoscaler := &types.Autoscaler{} if in.Autoscaler.Type == "" { autoscaler.Type = types.QueueDepthAutoscaler @@ -83,12 +52,27 @@ func (gws *GatewayService) GetOrCreateStub(ctx context.Context, in *pb.GetOrCrea in.Extra = "{}" } + if in.GpuCount > gws.appConfig.GatewayService.StubLimits.MaxGpuCount { + return &pb.GetOrCreateStubResponse{ + Ok: false, + ErrMsg: fmt.Sprintf("GPU count must be %d or less.", gws.appConfig.GatewayService.StubLimits.MaxGpuCount), + }, nil + } + + if in.GpuCount > 1 && !authInfo.Workspace.MultiGpuEnabled { + return &pb.GetOrCreateStubResponse{ + Ok: false, + ErrMsg: "Multi-GPU containers are not enabled for this workspace.", + }, nil + } + stubConfig := types.StubConfigV1{ Runtime: types.Runtime{ - Cpu: in.Cpu, - Gpus: gpus, - Memory: in.Memory, - ImageId: in.ImageId, + Cpu: in.Cpu, + Gpus: gpus, + GpuCount: in.GpuCount, + Memory: in.Memory, + ImageId: in.ImageId, }, Handler: in.Handler, OnStart: in.OnStart, @@ -104,6 +88,43 @@ func (gws *GatewayService) GetOrCreateStub(ctx context.Context, in *pb.GetOrCrea Authorized: in.Authorized, Autoscaler: autoscaler, Extra: json.RawMessage(in.Extra), + CheckpointEnabled: in.CheckpointEnabled, + } + + // Ensure GPU count is at least 1 if a GPU is required + if stubConfig.RequiresGPU() && in.GpuCount == 0 { + stubConfig.Runtime.GpuCount = 1 + } + + if stubConfig.RequiresGPU() { + concurrencyLimit, err := gws.backendRepo.GetConcurrencyLimitByWorkspaceId(ctx, authInfo.Workspace.ExternalId) + if err != nil && concurrencyLimit != nil && concurrencyLimit.GPULimit <= 0 { + return &pb.GetOrCreateStubResponse{ + Ok: false, + ErrMsg: "GPU concurrency limit is 0.", + }, nil + } + + gpuCounts, err := gws.providerRepo.GetGPUCounts(gws.appConfig.Worker.Pools) + if err != nil { + return &pb.GetOrCreateStubResponse{ + Ok: false, + ErrMsg: "Failed to get GPU counts.", + }, nil + } + + // T4s are currently in a different pool than other GPUs and won't show up in gpu counts + lowGpus := []string{} + + for _, gpu := range gpus { + if gpuCounts[gpu.String()] <= 1 && gpu.String() != types.GPU_T4.String() { + lowGpus = append(lowGpus, gpu.String()) + } + } + + if len(lowGpus) > 0 { + warning = fmt.Sprintf("GPU capacity for %s is currently low.", strings.Join(lowGpus, ", ")) + } } // Get secrets diff --git a/pkg/repository/backend_postgres.go b/pkg/repository/backend_postgres.go index e170235ff..e03a647e0 100644 --- a/pkg/repository/backend_postgres.go +++ b/pkg/repository/backend_postgres.go @@ -153,7 +153,7 @@ func (r *PostgresBackendRepository) CreateWorkspace(ctx context.Context) (types. func (r *PostgresBackendRepository) GetWorkspaceByExternalId(ctx context.Context, externalId string) (types.Workspace, error) { var workspace types.Workspace - query := `SELECT id, name, created_at, concurrency_limit_id, volume_cache_enabled FROM workspace WHERE external_id = $1;` + query := `SELECT id, name, created_at, concurrency_limit_id, volume_cache_enabled, multi_gpu_enabled FROM workspace WHERE external_id = $1;` err := r.client.GetContext(ctx, &workspace, query, externalId) if err != nil { return types.Workspace{}, err @@ -165,7 +165,7 @@ func (r *PostgresBackendRepository) GetWorkspaceByExternalId(ctx context.Context func (r *PostgresBackendRepository) GetWorkspaceByExternalIdWithSigningKey(ctx context.Context, externalId string) (types.Workspace, error) { var workspace types.Workspace - query := `SELECT id, name, created_at, concurrency_limit_id, signing_key, volume_cache_enabled FROM workspace WHERE external_id = $1;` + query := `SELECT id, name, created_at, concurrency_limit_id, signing_key, volume_cache_enabled, multi_gpu_enabled FROM workspace WHERE external_id = $1;` err := r.client.GetContext(ctx, &workspace, query, externalId) if err != nil { return types.Workspace{}, err @@ -209,7 +209,7 @@ func (r *PostgresBackendRepository) AuthorizeToken(ctx context.Context, tokenKey query := ` SELECT t.id, t.external_id, t.key, t.created_at, t.updated_at, t.active, t.disabled_by_cluster_admin , t.token_type, t.reusable, t.workspace_id, w.id "workspace.id", w.name "workspace.name", w.external_id "workspace.external_id", w.signing_key "workspace.signing_key", w.created_at "workspace.created_at", - w.updated_at "workspace.updated_at", w.volume_cache_enabled "workspace.volume_cache_enabled" + w.updated_at "workspace.updated_at", w.volume_cache_enabled "workspace.volume_cache_enabled", w.multi_gpu_enabled "workspace.multi_gpu_enabled" FROM token t INNER JOIN workspace w ON t.workspace_id = w.id WHERE t.key = $1 AND t.active = TRUE; @@ -558,7 +558,9 @@ func (c *PostgresBackendRepository) listTaskWithRelatedQueryBuilder(filters type } if len(filters.StubIds) > 0 { - qb = qb.Where(squirrel.Eq{"s.external_id": filters.StubIds}) + // Subquery to get the stub ids from the external ids + stubIdsSubquery := squirrel.Select("id").From("stub").Where(squirrel.Eq{"external_id": filters.StubIds}) + qb = qb.Where(squirrel.Expr("s.id in (?)", stubIdsSubquery)) } if len(filters.StubNames) > 0 { @@ -799,7 +801,7 @@ func (r *PostgresBackendRepository) GetStubByExternalId(ctx context.Context, ext var stub types.StubWithRelated qb := squirrel.StatementBuilder.PlaceholderFormat(squirrel.Dollar).Select( `s.id, s.external_id, s.name, s.type, s.config, s.config_version, s.object_id, s.workspace_id, s.created_at, s.updated_at, - w.id AS "workspace.id", w.external_id AS "workspace.external_id", w.name AS "workspace.name", w.created_at AS "workspace.created_at", w.updated_at AS "workspace.updated_at", w.signing_key AS "workspace.signing_key", w.volume_cache_enabled AS "workspace.volume_cache_enabled", + w.id AS "workspace.id", w.external_id AS "workspace.external_id", w.name AS "workspace.name", w.created_at AS "workspace.created_at", w.updated_at AS "workspace.updated_at", w.signing_key AS "workspace.signing_key", w.volume_cache_enabled AS "workspace.volume_cache_enabled", w.multi_gpu_enabled AS "workspace.multi_gpu_enabled", o.id AS "object.id", o.external_id AS "object.external_id", o.hash AS "object.hash", o.size AS "object.size", o.workspace_id AS "object.workspace_id", o.created_at AS "object.created_at"`, ). From("stub s"). diff --git a/pkg/repository/backend_postgres_migrations/018_add_stubid_index_task.go b/pkg/repository/backend_postgres_migrations/018_add_stubid_index_task.go new file mode 100644 index 000000000..dc2834806 --- /dev/null +++ b/pkg/repository/backend_postgres_migrations/018_add_stubid_index_task.go @@ -0,0 +1,22 @@ +package backend_postgres_migrations + +import ( + "context" + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationNoTxContext(upAddStubIdIndexTask, downAddStubIdIndexTask) +} + +func upAddStubIdIndexTask(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, `CREATE INDEX CONCURRENTLY IF NOT EXISTS task_stub_id on task(stub_id);`) + return err +} + +func downAddStubIdIndexTask(ctx context.Context, db *sql.DB) error { + _, err := db.ExecContext(ctx, `DROP INDEX CONCURRENTLY IF EXISTS task_stub_id `) + return err +} diff --git a/pkg/repository/backend_postgres_migrations/019_add_workspace_multi_gpu.go b/pkg/repository/backend_postgres_migrations/019_add_workspace_multi_gpu.go new file mode 100644 index 000000000..b2ffc8e83 --- /dev/null +++ b/pkg/repository/backend_postgres_migrations/019_add_workspace_multi_gpu.go @@ -0,0 +1,22 @@ +package backend_postgres_migrations + +import ( + "context" + "database/sql" + + "github.com/pressly/goose/v3" +) + +func init() { + goose.AddMigrationContext(upAddFieldWorkspaceMultiGpuEnabled, downDropFieldWorkspaceMultiGpuEnabled) +} + +func upAddFieldWorkspaceMultiGpuEnabled(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec(`ALTER TABLE workspace ADD COLUMN multi_gpu_enabled BOOLEAN DEFAULT FALSE;`) + return err +} + +func downDropFieldWorkspaceMultiGpuEnabled(ctx context.Context, tx *sql.Tx) error { + _, err := tx.Exec(`ALTER TABLE workspace DROP COLUMN multi_gpu_enabled;`) + return err +} diff --git a/pkg/repository/backend_postgres_test.go b/pkg/repository/backend_postgres_test.go new file mode 100644 index 000000000..b5247ba6b --- /dev/null +++ b/pkg/repository/backend_postgres_test.go @@ -0,0 +1,49 @@ +package repository + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/beam-cloud/beta9/pkg/types" + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/stretchr/testify/require" +) + +func TestListTaskWithRelated(t *testing.T) { + // Remove this skip if you are testing on local data + t.Skip() + ctx := context.Background() + + dbName := "control_plane" + dbUser := "pguser" + connStr := fmt.Sprintf("postgres://%s@localhost:5433/%s?sslmode=disable", dbUser, dbName) + db, err := sql.Open("postgres", connStr) + require.NoError(t, err) + client := sqlx.NewDb(db, "postgres") + require.NotNil(t, client) + r := PostgresBackendRepository{ + client: client, + } + + workspaceId := uint(0) + stubId := uuid.New().String() + firstPageId := uint(0) + secondPageId := uint(0) + + res, err := r.ListTasksWithRelatedPaginated(ctx, types.TaskFilter{WorkspaceID: workspaceId, StubIds: []string{stubId}}) + require.NoError(t, err) + require.NotNil(t, res) + require.Equal(t, 10, len(res.Data)) + require.Equal(t, firstPageId, res.Data[0].Id) + require.NotNil(t, res.Next) + + res, err = r.ListTasksWithRelatedPaginated(ctx, types.TaskFilter{WorkspaceID: workspaceId, StubIds: []string{stubId}, Cursor: res.Next}) + require.NoError(t, err) + require.NotNil(t, res) + require.Equal(t, 10, len(res.Data)) + require.Equal(t, secondPageId, res.Data[0].Id) + require.NotNil(t, res.Next) +} diff --git a/pkg/repository/base.go b/pkg/repository/base.go index 4d8ee4f89..5e04ee11c 100755 --- a/pkg/repository/base.go +++ b/pkg/repository/base.go @@ -56,6 +56,8 @@ type ContainerRepository interface { GetActiveContainersByWorkspaceId(workspaceId string) ([]types.ContainerState, error) GetActiveContainersByWorkerId(workerId string) ([]types.ContainerState, error) GetFailedContainerCountByStubId(stubId string) (int, error) + UpdateCheckpointState(workspaceName, checkpointId string, checkpointState *types.CheckpointState) error + GetCheckpointState(workspaceName, checkpointId string) (*types.CheckpointState, error) } type WorkspaceRepository interface { diff --git a/pkg/repository/container_redis.go b/pkg/repository/container_redis.go index 6b8d8eac9..f2c2be0fb 100644 --- a/pkg/repository/container_redis.go +++ b/pkg/repository/container_redis.go @@ -420,3 +420,39 @@ func (c *ContainerRedisRepository) SetContainerStateWithConcurrencyLimit(quota * return nil } + +func (cr *ContainerRedisRepository) UpdateCheckpointState(workspaceName, checkpointId string, checkpointState *types.CheckpointState) error { + stateKey := common.RedisKeys.SchedulerCheckpointState(workspaceName, checkpointId) + err := cr.rdb.HSet( + context.TODO(), stateKey, + "stub_id", checkpointState.StubId, + "container_id", checkpointState.ContainerId, + "status", string(checkpointState.Status), + "remote_key", checkpointState.RemoteKey, + ).Err() + if err != nil { + return fmt.Errorf("failed to set checkpoint state <%v>: %w", stateKey, err) + } + + return nil +} + +func (cr *ContainerRedisRepository) GetCheckpointState(workspaceName, checkpointId string) (*types.CheckpointState, error) { + stateKey := common.RedisKeys.SchedulerCheckpointState(workspaceName, checkpointId) + + res, err := cr.rdb.HGetAll(context.TODO(), stateKey).Result() + if err != nil && err != redis.Nil { + return nil, fmt.Errorf("failed to get container state: %w", err) + } + + if len(res) == 0 { + return nil, &types.ErrCheckpointNotFound{CheckpointId: checkpointId} + } + + state := &types.CheckpointState{} + if err = common.ToStruct(res, state); err != nil { + return nil, fmt.Errorf("failed to deserialize checkpoint state <%v>: %v", stateKey, err) + } + + return state, nil +} diff --git a/pkg/scheduler/pool_external.go b/pkg/scheduler/pool_external.go index 75150d866..f7fcbc450 100644 --- a/pkg/scheduler/pool_external.go +++ b/pkg/scheduler/pool_external.go @@ -376,6 +376,10 @@ func (wpc *ExternalWorkerPoolController) getWorkerEnvironment(workerId, machineI Name: "WORKER_ID", Value: workerId, }, + { + Name: "WORKER_POOL_NAME", + Value: wpc.name, + }, { Name: "CPU_LIMIT", Value: strconv.FormatInt(cpu, 10), diff --git a/pkg/scheduler/pool_local.go b/pkg/scheduler/pool_local.go index 1f981c97d..725759c8f 100644 --- a/pkg/scheduler/pool_local.go +++ b/pkg/scheduler/pool_local.go @@ -348,6 +348,10 @@ func (wpc *LocalKubernetesWorkerPoolController) getWorkerEnvironment(workerId st Name: "WORKER_ID", Value: workerId, }, + { + Name: "WORKER_POOL_NAME", + Value: wpc.name, + }, { Name: "CPU_LIMIT", Value: strconv.FormatInt(cpu, 10), diff --git a/pkg/scheduler/pool_manager.go b/pkg/scheduler/pool_manager.go index 741a3f773..309d4dc52 100644 --- a/pkg/scheduler/pool_manager.go +++ b/pkg/scheduler/pool_manager.go @@ -1,6 +1,9 @@ package scheduler import ( + "cmp" + "slices" + "github.com/beam-cloud/beta9/pkg/common" "github.com/beam-cloud/beta9/pkg/types" ) @@ -47,7 +50,8 @@ func (m *WorkerPoolManager) GetPoolByGPU(gpuType string) (*WorkerPool, bool) { } // GetPoolsByGPU retrieves all WorkerPools by their GPU type. -// It returns a slice of matching WorkerPools. +// It returns a slice of matching WorkerPools. The results are sorted by +// WorkerPoolConfig.Priority in descending order. func (m *WorkerPoolManager) GetPoolsByGPU(gpuType string) []*WorkerPool { var pools []*WorkerPool @@ -58,6 +62,10 @@ func (m *WorkerPoolManager) GetPoolsByGPU(gpuType string) []*WorkerPool { return true }) + slices.SortFunc(pools, func(a, b *WorkerPool) int { + return cmp.Compare(b.Config.Priority, a.Config.Priority) + }) + return pools } diff --git a/pkg/scheduler/pool_manager_test.go b/pkg/scheduler/pool_manager_test.go new file mode 100644 index 000000000..791f8e3c3 --- /dev/null +++ b/pkg/scheduler/pool_manager_test.go @@ -0,0 +1,57 @@ +package scheduler + +import ( + "testing" + + "github.com/beam-cloud/beta9/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestGetPoolsByGPU(t *testing.T) { + m := NewWorkerPoolManager() + + // Test that the pools are sorted by priority + pools := []*WorkerPool{ + { + Name: "aws-t4", + Config: types.WorkerPoolConfig{ + GPUType: "T4", + Priority: -1, + }, + Controller: &LocalWorkerPoolControllerForTest{}, + }, + { + Name: "gcp-t4", + Config: types.WorkerPoolConfig{ + GPUType: "T4", + Priority: 0, + }, + Controller: &LocalWorkerPoolControllerForTest{}, + }, + } + + for _, pool := range pools { + m.SetPool(pool.Name, pool.Config, pool.Controller) + } + + t4s := m.GetPoolsByGPU("T4") + assert.Equal(t, 2, len(t4s)) + assert.Equal(t, "gcp-t4", t4s[0].Name) // highest priority + assert.Equal(t, "aws-t4", t4s[1].Name) + + // When a new pool is added, it will still be sorted by priority + wp := &WorkerPool{ + Name: "xyz-t4", + Config: types.WorkerPoolConfig{ + GPUType: "T4", + Priority: 10, + }, + } + m.SetPool(wp.Name, wp.Config, wp.Controller) + + t4s = m.GetPoolsByGPU("T4") + assert.Equal(t, 3, len(t4s)) + assert.Equal(t, "xyz-t4", t4s[0].Name) // highest priority + assert.Equal(t, "gcp-t4", t4s[1].Name) + assert.Equal(t, "aws-t4", t4s[2].Name) +} diff --git a/pkg/task/dispatch.go b/pkg/task/dispatch.go index 62666afc1..b2161935d 100644 --- a/pkg/task/dispatch.go +++ b/pkg/task/dispatch.go @@ -186,7 +186,7 @@ func (d *Dispatcher) monitor(ctx context.Context) { } if !heartbeat { - d.retryTask(ctx, task, taskMessage) + d.RetryTask(ctx, task) continue } } @@ -194,7 +194,9 @@ func (d *Dispatcher) monitor(ctx context.Context) { } } -func (d *Dispatcher) retryTask(ctx context.Context, task types.TaskInterface, taskMessage *types.TaskMessage) error { +func (d *Dispatcher) RetryTask(ctx context.Context, task types.TaskInterface) error { + taskMessage := task.Message() + err := d.taskRepo.SetTaskRetryLock(ctx, taskMessage.WorkspaceName, taskMessage.StubId, taskMessage.TaskId) if err != nil { return err diff --git a/pkg/types/backend.go b/pkg/types/backend.go index ec99d5603..6a324b2fa 100644 --- a/pkg/types/backend.go +++ b/pkg/types/backend.go @@ -19,6 +19,7 @@ type Workspace struct { UpdatedAt time.Time `db:"updated_at" json:"updated_at,omitempty"` SigningKey *string `db:"signing_key" json:"signing_key"` VolumeCacheEnabled bool `db:"volume_cache_enabled" json:"volume_cache_enabled"` + MultiGpuEnabled bool `db:"multi_gpu_enabled" json:"multi_gpu_enabled"` ConcurrencyLimitId *uint `db:"concurrency_limit_id" json:"concurrency_limit_id,omitempty"` ConcurrencyLimit *ConcurrencyLimit `db:"concurrency_limit" json:"concurrency_limit"` } @@ -183,6 +184,11 @@ type StubConfigV1 struct { Secrets []Secret `json:"secrets,omitempty"` Autoscaler *Autoscaler `json:"autoscaler"` Extra json.RawMessage `json:"extra"` + CheckpointEnabled bool `json:"checkpoint_enabled"` +} + +func (c *StubConfigV1) RequiresGPU() bool { + return len(c.Runtime.Gpus) > 0 || c.Runtime.Gpu != "" } type AutoscalerType string @@ -290,11 +296,12 @@ type Image struct { } type Runtime struct { - Cpu int64 `json:"cpu"` - Gpu GpuType `json:"gpu"` - Memory int64 `json:"memory"` - ImageId string `json:"image_id"` - Gpus []GpuType `json:"gpus"` + Cpu int64 `json:"cpu"` + Gpu GpuType `json:"gpu"` + GpuCount uint32 `json:"gpu_count"` + Memory int64 `json:"memory"` + ImageId string `json:"image_id"` + Gpus []GpuType `json:"gpus"` } type GpuType string diff --git a/pkg/types/config.go b/pkg/types/config.go index 2993bc32a..0c6def24a 100644 --- a/pkg/types/config.go +++ b/pkg/types/config.go @@ -4,6 +4,7 @@ import ( "time" blobcache "github.com/beam-cloud/blobcache-v2/pkg" + cedana "github.com/cedana/cedana/pkg/types" corev1 "k8s.io/api/core/v1" ) @@ -88,6 +89,7 @@ type CORSConfig struct { type StubLimits struct { Memory uint64 `key:"memory" json:"memory"` MaxReplicas uint64 `key:"maxReplicas" json:"max_replicas"` + MaxGpuCount uint32 `key:"maxGpuCount" json:"max_gpu_count"` } type GatewayServiceConfig struct { @@ -201,6 +203,7 @@ type WorkerConfig struct { AddWorkerTimeout time.Duration `key:"addWorkerTimeout" json:"add_worker_timeout"` TerminationGracePeriod int64 `key:"terminationGracePeriod"` BlobCacheEnabled bool `key:"blobCacheEnabled" json:"blob_cache_enabled"` + CRIU CRIUConfig `key:"criu" json:"criu"` } type PoolMode string @@ -222,6 +225,7 @@ type WorkerPoolConfig struct { Priority int32 `key:"priority" json:"priority"` Preemptable bool `key:"preemptable" json:"preemptable"` UserData string `key:"userData" json:"user_data"` + CRIUEnabled bool `key:"criuEnabled" json:"criu_enabled"` } type WorkerPoolJobSpecConfig struct { @@ -386,6 +390,15 @@ type FluentBitEventMapping struct { Tag string `key:"tag" json:"tag"` } +type ObjectStoreConfig struct { + BucketName string `key:"bucketName" json:"bucket_name"` + AccessKey string `key:"accessKey" json:"access_key"` + SecretKey string `key:"secretKey" json:"secret_key"` + EndpointURL string `key:"endpointURL" json:"bucket_url"` + Region string `key:"region" json:"region"` + ReadOnly bool `key:"readOnly" json:"read_only"` +} + type FluentBitEventConfig struct { Endpoint string `key:"endpoint" json:"endpoint"` MaxConns int `key:"maxConns" json:"max_conns"` @@ -396,6 +409,24 @@ type FluentBitEventConfig struct { Mapping []FluentBitEventMapping `key:"mapping" json:"mapping"` } +type CRIUConfig struct { + Storage CheckpointStorageConfig `key:"storage" json:"storage"` + Cedana cedana.Config `key:"cedana" json:"cedana"` +} + +type CheckpointStorageConfig struct { + MountPath string `key:"mountPath" json:"mount_path"` + Mode string `key:"mode" json:"mode"` + ObjectStore ObjectStoreConfig `key:"objectStoreConfig" json:"object_store_config"` +} + +type CheckpointStorageMode string + +var ( + CheckpointStorageModeLocal CheckpointStorageMode = "local" + CheckpointStorageModeS3 CheckpointStorageMode = "s3" +) + type AbstractionConfig struct { Bot BotConfig `key:"bot" json:"bot"` } diff --git a/pkg/types/scheduler.go b/pkg/types/scheduler.go index f76c230ed..1cb652f40 100644 --- a/pkg/types/scheduler.go +++ b/pkg/types/scheduler.go @@ -74,30 +74,31 @@ type ContainerState struct { } type ContainerRequest struct { - ContainerId string `json:"container_id"` - EntryPoint []string `json:"entry_point"` - Env []string `json:"env"` - Cpu int64 `json:"cpu"` - Memory int64 `json:"memory"` - Gpu string `json:"gpu"` - GpuRequest []string `json:"gpu_request"` - GpuCount uint32 `json:"gpu_count"` - SourceImage *string `json:"source_image"` - SourceImageCreds string `json:"source_image_creds"` - ImageId string `json:"image_id"` - StubId string `json:"stub_id"` - WorkspaceId string `json:"workspace_id"` - Workspace Workspace `json:"workspace"` - Stub StubWithRelated `json:"stub"` - Timestamp time.Time `json:"timestamp"` - Mounts []Mount `json:"mounts"` - RetryCount int `json:"retry_count"` - PoolSelector string `json:"pool_selector"` - Preemptable bool `json:"preemptable"` + ContainerId string `json:"container_id"` + EntryPoint []string `json:"entry_point"` + Env []string `json:"env"` + Cpu int64 `json:"cpu"` + Memory int64 `json:"memory"` + Gpu string `json:"gpu"` + GpuRequest []string `json:"gpu_request"` + GpuCount uint32 `json:"gpu_count"` + SourceImage *string `json:"source_image"` + SourceImageCreds string `json:"source_image_creds"` + ImageId string `json:"image_id"` + StubId string `json:"stub_id"` + WorkspaceId string `json:"workspace_id"` + Workspace Workspace `json:"workspace"` + Stub StubWithRelated `json:"stub"` + Timestamp time.Time `json:"timestamp"` + Mounts []Mount `json:"mounts"` + RetryCount int `json:"retry_count"` + PoolSelector string `json:"pool_selector"` + Preemptable bool `json:"preemptable"` + CheckpointEnabled bool `json:"checkpoint_enabled"` } func (c *ContainerRequest) RequiresGPU() bool { - return len(c.GpuRequest) > 0 + return len(c.GpuRequest) > 0 || c.Gpu != "" } const ContainerExitCodeTtlS int = 300 @@ -176,6 +177,30 @@ func (e *QuotaDoesNotExistError) Error() string { return "quota_does_not_exist" } +type CheckpointStatus string + +const ( + CheckpointStatusAvailable CheckpointStatus = "available" + CheckpointStatusCheckpointFailed CheckpointStatus = "checkpoint_failed" + CheckpointStatusRestoreFailed CheckpointStatus = "restore_failed" + CheckpointStatusNotFound CheckpointStatus = "not_found" +) + +type CheckpointState struct { + StubId string `redis:"stub_id" json:"stub_id"` + ContainerId string `redis:"container_id" json:"container_id"` + Status CheckpointStatus `redis:"status" json:"status"` + RemoteKey string `redis:"remote_key" json:"remote_key"` +} + +type ErrCheckpointNotFound struct { + CheckpointId string +} + +func (e *ErrCheckpointNotFound) Error() string { + return fmt.Sprintf("checkpoint state not found: %s", e.CheckpointId) +} + type StopContainerArgs struct { ContainerId string `json:"container_id"` Force bool `json:"force"` diff --git a/pkg/types/task.go b/pkg/types/task.go index 9f1d27e2f..9acf9bbd0 100644 --- a/pkg/types/task.go +++ b/pkg/types/task.go @@ -36,6 +36,7 @@ type TaskInterface interface { Retry(ctx context.Context) error HeartBeat(ctx context.Context) (bool, error) Metadata() TaskMetadata + Message() *TaskMessage } type TaskExecutor string diff --git a/pkg/worker/base_runc_config.json b/pkg/worker/base_runc_config.json index d89cc80b8..adaa9562f 100644 --- a/pkg/worker/base_runc_config.json +++ b/pkg/worker/base_runc_config.json @@ -81,7 +81,8 @@ { "destination": "/proc", "type": "proc", - "source": "proc" + "source": "proc", + "options": ["rw", "nosuid", "noexec", "nodev"] }, { "destination": "/root", @@ -108,6 +109,7 @@ "type": "tmpfs", "source": "tmpfs", "options": [ + "rw", "nosuid", "strictatime", "mode=755", @@ -154,10 +156,10 @@ "type": "sysfs", "source": "sysfs", "options": [ + "rw", "nosuid", "noexec", - "nodev", - "ro" + "nodev" ] }, { @@ -168,8 +170,7 @@ "nosuid", "noexec", "nodev", - "relatime", - "ro" + "relatime" ] }, { @@ -434,37 +435,11 @@ ] }, "namespaces": [ - { - "type": "pid" - }, - { - "type": "ipc" - }, - { - "type": "uts" - }, - { - "type": "mount" - } - ], - "maskedPaths": [ - "/proc/acpi", - "/proc/asound", - "/proc/kcore", - "/proc/keys", - "/proc/latency_stats", - "/proc/timer_list", - "/proc/timer_stats", - "/proc/sched_debug", - "/sys/firmware", - "/proc/scsi" - ], - "readonlyPaths": [ - "/proc/bus", - "/proc/fs", - "/proc/irq", - "/proc/sys", - "/proc/sysrq-trigger" + { "type": "mount" }, + { "type": "uts" }, + { "type": "pid" }, + { "type": "ipc" }, + { "type": "cgroup" } ] } -} \ No newline at end of file +} diff --git a/pkg/worker/cedana.go b/pkg/worker/cedana.go new file mode 100644 index 000000000..bdc189cf9 --- /dev/null +++ b/pkg/worker/cedana.go @@ -0,0 +1,330 @@ +package worker + +import ( + "context" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "time" + + cedanagrpc "buf.build/gen/go/cedana/task/grpc/go/_gogrpc" + cedanaproto "buf.build/gen/go/cedana/task/protocolbuffers/go" + "github.com/beam-cloud/go-runc" + types "github.com/cedana/cedana/pkg/types" + + "github.com/opencontainers/runtime-spec/specs-go" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" +) + +const ( + runcRoot = "/run/runc" + cedanaHost = "0.0.0.0" + cedanaBinPath = "/usr/bin/cedana" + cedanaSharedLibPath = "/usr/local/lib/libcedana-gpu.so" + cedanaLogLevel = "info" + checkpointPathBase = "/tmp/checkpoints" + defaultManageDeadline = 10 * time.Second + defaultCheckpointDeadline = 10 * time.Minute + defaultRestoreDeadline = 5 * time.Minute + defaultHealthCheckDeadline = 30 * time.Second + cedanaUseRemoteDB = true // Do not change, or migrations across workers will fail +) + +type CedanaClient struct { + conn *grpc.ClientConn + service cedanagrpc.TaskServiceClient + daemon *exec.Cmd + config types.Config +} + +func NewCedanaClient( + ctx context.Context, + config types.Config, + gpuEnabled bool, +) (*CedanaClient, error) { + var opts []grpc.DialOption + opts = append(opts, grpc.WithTransportCredentials(insecure.NewCredentials())) + + port, err := getRandomFreePort() + if err != nil { + return nil, err + } + + addr := fmt.Sprintf("%s:%d", cedanaHost, port) + taskConn, err := grpc.NewClient(addr, opts...) + if err != nil { + return nil, err + } + + taskClient := cedanagrpc.NewTaskServiceClient(taskConn) + + // Launch the daemon + daemon := exec.CommandContext(ctx, cedanaBinPath, "daemon", "start", + fmt.Sprintf("--port=%d", port), + fmt.Sprintf("--gpu-enabled=%t", gpuEnabled)) + + daemon.Stdout = os.Stdout + daemon.Stderr = os.Stderr + + // XXX: Set config using env until config JSON parsing is fixed + daemon.Env = append(os.Environ(), + fmt.Sprintf("CEDANA_LOG_LEVEL=%s", cedanaLogLevel), + fmt.Sprintf("CEDANA_CLIENT_LEAVE_RUNNING=%t", config.Client.LeaveRunning), + fmt.Sprintf("CEDANA_DUMP_STORAGE_DIR=%s", config.SharedStorage.DumpStorageDir), + fmt.Sprintf("CEDANA_URL=%s", config.Connection.CedanaUrl), + fmt.Sprintf("CEDANA_AUTH_TOKEN=%s", config.Connection.CedanaAuthToken), + fmt.Sprintf("CEDANA_REMOTE=%t", cedanaUseRemoteDB), + ) + + err = daemon.Start() + if err != nil { + return nil, fmt.Errorf("failed to start cedana daemon: %v", err) + } + + // Cleanup the daemon on exit + go func() { + daemon.Wait() + taskConn.Close() + }() + + client := &CedanaClient{ + service: taskClient, + conn: taskConn, + daemon: daemon, + config: config, + } + + // Wait for the daemon to be ready, and do health check + details, err := client.DetailedHealthCheckWait(ctx) + if err != nil || len(details.UnhealthyReasons) > 0 { + defer daemon.Process.Kill() + defer taskConn.Close() + + if err != nil { + return nil, fmt.Errorf("cedana health check failed: %v", err) + } + + if len(details.UnhealthyReasons) > 0 { + return nil, fmt.Errorf( + "cedana health failed with reasons: %v", + details.UnhealthyReasons, + ) + } + } + + return client, nil +} + +func (c *CedanaClient) Close() { + c.conn.Close() + c.daemon.Process.Kill() +} + +// Updates the runc container spec to make the shared library available +// as well as the shared memory that is used for communication +func (c *CedanaClient) PrepareContainerSpec(spec *specs.Spec, containerId string, containerHostname string, gpuEnabled bool) error { + os.MkdirAll(checkpointSignalDir(containerId), os.ModePerm) // Add a mount point for the checkpoint signal file + + spec.Mounts = append(spec.Mounts, specs.Mount{ + Type: "bind", + Source: checkpointSignalDir(containerId), + Destination: "/cedana", + Options: []string{ + "rbind", + "rprivate", + "nosuid", + "nodev", + }, + }) + + containerIdPath := filepath.Join(checkpointSignalDir(containerId), checkpointContainerIdFileName) + err := os.WriteFile(containerIdPath, []byte(containerId), 0644) + if err != nil { + return err + } + + containerHostnamePath := filepath.Join(checkpointSignalDir(containerId), checkpointContainerHostnameFileName) + err = os.WriteFile(containerHostnamePath, []byte(containerHostname), 0644) + if err != nil { + return err + } + + if !gpuEnabled { + return nil // No need to do anything else if GPU is not enabled + } + + // First check if shared library is on worker + if _, err := os.Stat(cedanaSharedLibPath); os.IsNotExist(err) { + return fmt.Errorf( + "%s not found on worker. Was the daemon started with GPU enabled?", + cedanaSharedLibPath, + ) + } + + // Remove nvidia prestart hook as we don't need actual device mounts + spec.Hooks.Prestart = nil + + // Add shared memory mount from worker instead, remove existing /dev/shm mount + for i, m := range spec.Mounts { + if m.Destination == "/dev/shm" { + spec.Mounts = append(spec.Mounts[:i], spec.Mounts[i+1:]...) + break + } + } + + // Add shared memory mount from worker + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: "/dev/shm", + Source: "/dev/shm", + Type: "bind", + Options: []string{ + "rbind", + "rprivate", + "nosuid", + "nodev", + "rw", + }, + }) + + // Add the shared library to the container + spec.Mounts = append(spec.Mounts, specs.Mount{ + Destination: cedanaSharedLibPath, + Source: cedanaSharedLibPath, + Type: "bind", + Options: []string{ + "rbind", + "rprivate", + "nosuid", + "nodev", + "rw", + }, + }) + + // XXX: Remove /usr/lib/worker/x86_64-linux-gnu from mounts + for i, m := range spec.Mounts { + if m.Destination == "/usr/lib/worker/x86_64-linux-gnu" { + spec.Mounts = append(spec.Mounts[:i], spec.Mounts[i+1:]...) + break + } + } + + spec.Process.Env = append(spec.Process.Env, "CEDANA_JID="+containerId, "LD_PRELOAD="+cedanaSharedLibPath) + return nil +} + +// Start managing a runc container +func (c *CedanaClient) Manage(ctx context.Context, containerId string, gpuEnabled bool) error { + ctx, cancel := context.WithTimeout(ctx, defaultManageDeadline) + defer cancel() + + args := &cedanaproto.RuncManageArgs{ + ContainerID: containerId, + GPU: gpuEnabled, + Root: runcRoot, + } + _, err := c.service.RuncManage(ctx, args) + if err != nil { + return err + } + return nil +} + +// Checkpoint a runc container, returns the path to the checkpoint +func (c *CedanaClient) Checkpoint(ctx context.Context, containerId string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, defaultCheckpointDeadline) + defer cancel() + + args := cedanaproto.JobDumpArgs{ + JID: containerId, + CriuOpts: &cedanaproto.CriuOpts{ + TcpClose: true, + TcpEstablished: true, + LeaveRunning: true, + TcpSkipInFlight: true, + }, + Dir: fmt.Sprintf("%s/%s", checkpointPathBase, containerId), + } + res, err := c.service.JobDump(ctx, &args) + if err != nil { + return "", err + } + return res.GetState().GetCheckpointPath(), nil +} + +type cedanaRestoreOpts struct { + jobId string + containerId string + checkpointPath string + cacheFunc func(string, string) (string, error) +} + +// Restore a runc container. If a checkpoint path is provided, it will be used as the checkpoint. +// If empty path is provided, the latest checkpoint path from DB will be used. +func (c *CedanaClient) Restore( + ctx context.Context, + restoreOpts cedanaRestoreOpts, + runcOpts *runc.CreateOpts, +) (*cedanaproto.ProcessState, error) { + ctx, cancel := context.WithTimeout(ctx, defaultCheckpointDeadline) + defer cancel() + + // NOTE: Cedana uses bundle path to find the config.json + bundle := strings.TrimRight(runcOpts.ConfigPath, filepath.Base(runcOpts.ConfigPath)) + + // If a cache function is provided, attempt to cache the checkpoint nearby + if restoreOpts.cacheFunc != nil { + checkpointPath, err := restoreOpts.cacheFunc(restoreOpts.containerId, restoreOpts.checkpointPath) + if err == nil { + log.Printf("<%s> - using cached checkpoint located at: %s\n", restoreOpts.containerId, checkpointPath) + restoreOpts.checkpointPath = checkpointPath + } else { + log.Printf("<%s> - failed to cache checkpoint nearby: %v\n", restoreOpts.containerId, err) + } + } + + args := &cedanaproto.JobRestoreArgs{ + JID: restoreOpts.jobId, + RuncOpts: &cedanaproto.RuncOpts{ + Root: runcRoot, + Bundle: bundle, + Detach: true, + ConsoleSocket: runcOpts.ConsoleSocket.Path(), + ContainerID: restoreOpts.containerId, + }, + CriuOpts: &cedanaproto.CriuOpts{TcpClose: true, TcpEstablished: true}, + CheckpointPath: restoreOpts.checkpointPath, + } + res, err := c.service.JobRestore(ctx, args) + if err != nil { + return nil, err + } + + if runcOpts.Started != nil { + runcOpts.Started <- int(res.GetState().GetPID()) + } + + return res.State, nil +} + +// Perform a detailed health check of cedana C/R capabilities +func (c *CedanaClient) DetailedHealthCheckWait( + ctx context.Context, +) (*cedanaproto.DetailedHealthCheckResponse, error) { + ctx, cancel := context.WithTimeout(ctx, defaultHealthCheckDeadline) + defer cancel() + + opts := []grpc.CallOption{} + opts = append(opts, grpc.WaitForReady(true)) + + res, err := c.service.DetailedHealthCheck(ctx, &cedanaproto.DetailedHealthCheckRequest{}, opts...) + if err != nil { + return nil, err + } + + return res, nil +} diff --git a/pkg/worker/cr.go b/pkg/worker/cr.go new file mode 100644 index 000000000..0fa17e688 --- /dev/null +++ b/pkg/worker/cr.go @@ -0,0 +1,257 @@ +package worker + +import ( + "context" + _ "embed" + "fmt" + "log" + "log/slog" + "os" + "path/filepath" + "time" + + common "github.com/beam-cloud/beta9/pkg/common" + types "github.com/beam-cloud/beta9/pkg/types" + "github.com/beam-cloud/go-runc" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func (s *Worker) attemptCheckpointOrRestore(ctx context.Context, request *types.ContainerRequest, consoleWriter *ConsoleWriter, startedChan chan int, configPath string) (bool, string, error) { + state, createCheckpoint := s.shouldCreateCheckpoint(request) + + // If checkpointing is enabled, attempt to create a checkpoint + if createCheckpoint { + go func() { + err := s.createCheckpoint(ctx, request) + if err != nil { + log.Printf("<%s> - failed to create checkpoint: %v\n", request.ContainerId, err) + } + }() + } else if state.Status == types.CheckpointStatusAvailable { + checkpointPath := filepath.Join(s.config.Worker.CRIU.Storage.MountPath, state.RemoteKey) + + os.Create(filepath.Join(checkpointSignalDir(request.ContainerId), checkpointCompleteFileName)) + + _, err := s.cedanaClient.Restore(ctx, cedanaRestoreOpts{ + checkpointPath: checkpointPath, + jobId: state.ContainerId, + containerId: request.ContainerId, + cacheFunc: s.cacheCheckpoint, + }, &runc.CreateOpts{ + Detach: true, + ConsoleSocket: consoleWriter, + ConfigPath: configPath, + Started: startedChan, + }) + + if err != nil { + log.Printf("<%s> - failed to restore checkpoint: %v\n", request.ContainerId, err) + + s.containerRepo.UpdateCheckpointState(request.Workspace.Name, request.StubId, &types.CheckpointState{ + Status: types.CheckpointStatusRestoreFailed, + ContainerId: request.ContainerId, + StubId: request.StubId, + }) + + return false, "", err + } else { + log.Printf("<%s> - checkpoint found and restored \n", request.ContainerId) + return true, state.ContainerId, nil + } + } + + return false, "", nil +} + +// Waits for the container to be ready to checkpoint at the desired point in execution, ie. +// after all processes within a container have reached a checkpointable state +func (s *Worker) createCheckpoint(ctx context.Context, request *types.ContainerRequest) error { + os.MkdirAll(filepath.Join(s.config.Worker.CRIU.Storage.MountPath, request.Workspace.Name), os.ModePerm) + + timeout := defaultCheckpointDeadline + managing := false + gpuEnabled := request.RequiresGPU() + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + +waitForReady: + for { + select { + case <-ctx.Done(): + return fmt.Errorf("checkpoint deadline exceeded or container exited") + case <-ticker.C: + instance, exists := s.containerInstances.Get(request.ContainerId) + if !exists { + log.Printf("<%s> - container instance not found yet\n", request.ContainerId) + continue + } + + // Start managing the container with Cedana + if !managing { + err := s.cedanaClient.Manage(ctx, instance.Id, gpuEnabled) + if err == nil { + managing = true + } else { + log.Printf("<%s> - cedana manage failed, container may not be started yet: %+v\n", instance.Id, err) + } + + continue + } + + // Check if the container is ready for checkpoint by verifying the existence of a signal file + readyFilePath := filepath.Join(checkpointSignalDir(instance.Id), checkpointSignalFileName) + if _, err := os.Stat(readyFilePath); err == nil { + log.Printf("<%s> - container ready for checkpoint.\n", instance.Id) + break waitForReady + } else { + log.Printf("<%s> - container not ready for checkpoint.\n", instance.Id) + } + + } + } + + // Proceed to create the checkpoint + checkpointPath, err := s.cedanaClient.Checkpoint(ctx, request.ContainerId) + if err != nil { + s.containerRepo.UpdateCheckpointState(request.Workspace.Name, request.StubId, &types.CheckpointState{ + Status: types.CheckpointStatusCheckpointFailed, + ContainerId: request.ContainerId, + StubId: request.StubId, + }) + return err + } + + // Create a file accessible to the container to indicate that the checkpoint has been captured + os.Create(filepath.Join(checkpointSignalDir(request.ContainerId), checkpointCompleteFileName)) + + // Move compressed checkpoint file to long-term storage directory + remoteKey := filepath.Join(request.Workspace.Name, filepath.Base(checkpointPath)) + err = copyFile(checkpointPath, filepath.Join(s.config.Worker.CRIU.Storage.MountPath, remoteKey)) + if err != nil { + return err + } + + err = os.Remove(checkpointPath) + if err != nil { + log.Printf("<%s> - failed to delete temporary checkpoint file: %v\n", request.ContainerId, err) + } + + log.Printf("<%s> - checkpoint created successfully\n", request.ContainerId) + + return s.containerRepo.UpdateCheckpointState(request.Workspace.Name, request.StubId, &types.CheckpointState{ + Status: types.CheckpointStatusAvailable, + ContainerId: request.ContainerId, // We store this as a reference to the container that we initially checkpointed + StubId: request.StubId, + RemoteKey: remoteKey, + }) +} + +// shouldCreateCheckpoint checks if a checkpoint should be created for a given container +// NOTE: this currently only works for deployments since functions can run multiple containers +func (s *Worker) shouldCreateCheckpoint(request *types.ContainerRequest) (types.CheckpointState, bool) { + if !s.IsCRIUAvailable() || !request.CheckpointEnabled { + return types.CheckpointState{}, false + } + + state, err := s.containerRepo.GetCheckpointState(request.Workspace.Name, request.StubId) + if err != nil { + if _, ok := err.(*types.ErrCheckpointNotFound); !ok { + return types.CheckpointState{}, false + } + + // If checkpoint not found, we can proceed to create one + return types.CheckpointState{Status: types.CheckpointStatusNotFound}, true + } + + return *state, false +} + +// Wait for a restored container to exit +func (s *Worker) waitForRestoredContainer(ctx context.Context, containerId string, startedChan chan int, logChan chan common.LogRecord, request *types.ContainerRequest, spec *specs.Spec) int { + pid := <-startedChan + outputLogger := slog.New(common.NewChannelHandler(logChan)) + + // Clean up runc container state and send final output message + cleanup := func(exitCode int, err error) int { + log.Printf("<%s> - container has exited with code: %d\n", containerId, exitCode) + + outputLogger.Info("", "done", true, "success", exitCode == 0) + + err = s.runcHandle.Delete(s.ctx, containerId, &runc.DeleteOpts{Force: true}) + if err != nil { + log.Printf("<%s> - failed to delete container: %v\n", containerId, err) + } + + return exitCode + } + + go s.collectAndSendContainerMetrics(ctx, request, spec, pid) // Capture resource usage (cpu/mem/gpu) + go s.watchOOMEvents(ctx, containerId, outputLogger) // Watch for OOM events + + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return cleanup(0, nil) + case <-ticker.C: + state, err := s.runcHandle.State(ctx, containerId) + if err != nil { + return cleanup(-1, err) + } + + if state.Status != types.RuncContainerStatusRunning && state.Status != types.RuncContainerStatusPaused { + return cleanup(0, nil) + } + } + } +} + +// Caches a checkpoint nearby if the file cache is available +func (s *Worker) cacheCheckpoint(containerId, checkpointPath string) (string, error) { + cachedCheckpointPath := filepath.Join(baseFileCachePath, checkpointPath) + + if s.fileCacheManager.CacheAvailable() { + + // If the checkpoint is already cached, we can use that path without the extra grpc call + if _, err := os.Stat(cachedCheckpointPath); err == nil { + return cachedCheckpointPath, nil + } + + log.Printf("<%s> - caching checkpoint nearby: %s\n", containerId, checkpointPath) + client := s.fileCacheManager.GetClient() + + // Remove the leading "/" from the checkpoint path + _, err := client.StoreContentFromSource(checkpointPath[1:], 0) + if err != nil { + return "", err + } + + checkpointPath = cachedCheckpointPath + } + + return checkpointPath, nil +} + +func (s *Worker) IsCRIUAvailable() bool { + if s.cedanaClient == nil { + return false + } + + poolName := os.Getenv("WORKER_POOL_NAME") + if poolName == "" { + return false + } + + pool, ok := s.config.Worker.Pools[poolName] + if !ok { + return false + } + + return pool.CRIUEnabled +} diff --git a/pkg/worker/image.go b/pkg/worker/image.go index fc9444f8a..52bfc50b9 100644 --- a/pkg/worker/image.go +++ b/pkg/worker/image.go @@ -142,6 +142,8 @@ func (c *ImageClient) PullLazy(request *types.ContainerRequest) error { _, err := c.cacheClient.StoreContentFromSource(sourcePath, sourceOffset) if err == nil { localCachePath = baseBlobFsContentPath + } else { + c.logger.Log(request.ContainerId, request.StubId, "unable to cache image nearby <%s>: %v\n", imageId, err) } } } diff --git a/pkg/worker/lifecycle.go b/pkg/worker/lifecycle.go index be8316013..ce2ceab4e 100644 --- a/pkg/worker/lifecycle.go +++ b/pkg/worker/lifecycle.go @@ -5,6 +5,7 @@ import ( _ "embed" "encoding/json" "fmt" + "log" "log/slog" "os" "path/filepath" @@ -17,23 +18,21 @@ import ( types "github.com/beam-cloud/beta9/pkg/types" runc "github.com/beam-cloud/go-runc" "github.com/opencontainers/runtime-spec/specs-go" - "github.com/rs/zerolog/log" ) const ( - baseConfigPath string = "/tmp" - defaultContainerDirectory string = "/mnt/code" - specBaseName string = "config.json" - initialSpecBaseName string = "initial_config.json" - - exitCodeSigterm int = 143 // Means the container received a SIGTERM by the underlying operating system - exitCodeSigkill int = 137 // Means the container received a SIGKILL by the underlying operating system + baseConfigPath string = "/tmp" + defaultContainerDirectory string = "/mnt/code" + specBaseName string = "config.json" + initialSpecBaseName string = "initial_config.json" + runcEventsInterval time.Duration = 5 * time.Second + containerInnerPort int = 8001 // Use a fixed port inside the container + exitCodeSigterm int = 143 // Means the container received a SIGTERM by the underlying operating system + exitCodeSigkill int = 137 // Means the container received a SIGKILL by the underlying operating system ) -var ( - //go:embed base_runc_config.json - baseRuncConfigRaw string -) +//go:embed base_runc_config.json +var baseRuncConfigRaw string // handleStopContainerEvent used by the event bus to stop a container. // Containers are stopped by sending a stop container event to stopContainerChan. @@ -43,12 +42,12 @@ func (s *Worker) handleStopContainerEvent(event *common.Event) bool { stopArgs, err := types.ToStopContainerArgs(event.Args) if err != nil { - log.Error().Err(err).Msg("failed to parse stop container args") + log.Printf("failed to parse stop container args: %v\n", err) return false } if _, exists := s.containerInstances.Get(stopArgs.ContainerId); exists { - log.Info().Str("container_id", stopArgs.ContainerId).Msg("received stop container event") + log.Printf("<%s> - received stop container event.\n", stopArgs.ContainerId) s.stopContainerChan <- stopContainerEvent{ContainerId: stopArgs.ContainerId, Kill: stopArgs.Force} } @@ -57,11 +56,11 @@ func (s *Worker) handleStopContainerEvent(event *common.Event) bool { // stopContainer stops a runc container. When force is true, a SIGKILL signal is sent to the container. func (s *Worker) stopContainer(containerId string, kill bool) error { - log.Info().Str("container_id", containerId).Msg("stopping container") + log.Printf("<%s> - stopping container.\n", containerId) - _, exists := s.containerInstances.Get(containerId) + instance, exists := s.containerInstances.Get(containerId) if !exists { - log.Info().Str("container_id", containerId).Msg("container not found") + log.Printf("<%s> - container not found.\n", containerId) return nil } @@ -70,14 +69,14 @@ func (s *Worker) stopContainer(containerId string, kill bool) error { signal = int(syscall.SIGKILL) } - err := s.runcHandle.Kill(context.Background(), containerId, signal, &runc.KillOpts{All: true}) + err := s.runcHandle.Kill(context.Background(), instance.Id, signal, &runc.KillOpts{All: true}) if err != nil { - log.Info().Str("container_id", containerId).Msgf("error stopping container: %v", err) + log.Printf("<%s> - error stopping container: %v\n", containerId, err) s.containerNetworkManager.TearDown(containerId) return nil } - log.Info().Str("container_id", containerId).Msg("container stopped") + log.Printf("<%s> - container stopped.\n", containerId) return nil } @@ -92,7 +91,7 @@ func (s *Worker) finalizeContainer(containerId string, request *types.ContainerR err := s.containerRepo.SetContainerExitCode(containerId, *exitCode) if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to set exit code") + log.Printf("<%s> - failed to set exit code: %v\n", containerId, err) } s.clearContainer(containerId, request, *exitCode) @@ -109,7 +108,7 @@ func (s *Worker) clearContainer(containerId string, request *types.ContainerRequ // Tear down container network components err := s.containerNetworkManager.TearDown(request.ContainerId) if err != nil { - log.Error().Str("container_id", request.ContainerId).Err(err).Msg("failed to clean up container network") + log.Printf("<%s> - failed to clean up container network: %v\n", request.ContainerId, err) } s.completedRequests <- request @@ -134,17 +133,17 @@ func (s *Worker) clearContainer(containerId string, request *types.ContainerRequ container, err := s.runcHandle.State(context.TODO(), containerId) if err == nil && container.Status == types.RuncContainerStatusRunning { if err := s.stopContainer(containerId, true); err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to stop container") + log.Printf("<%s> - failed to stop container: %v\n", containerId, err) } } s.containerInstances.Delete(containerId) err = s.containerRepo.DeleteContainerState(containerId) if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to remove container state") + log.Printf("<%s> - failed to remove container state: %v\n", containerId, err) } - log.Info().Str("container_id", containerId).Msg("finalized container shutdown") + log.Printf("<%s> - finalized container shutdown.\n", containerId) }() } @@ -154,10 +153,10 @@ func (s *Worker) RunContainer(request *types.ContainerRequest) error { bundlePath := filepath.Join(s.imageMountPath, request.ImageId) // Pull image - log.Info().Str("container_id", containerId).Str("image_id", request.ImageId).Msg("lazy-pulling image") + log.Printf("<%s> - lazy-pulling image: %s\n", containerId, request.ImageId) err := s.imageClient.PullLazy(request) if err != nil && request.SourceImage != nil { - log.Info().Str("container_id", containerId).Str("image_id", *request.SourceImage).Msg("lazy-pull failed, pulling source image") + log.Printf("<%s> - lazy-pull failed, pulling source image: %s\n", containerId, *request.SourceImage) err = s.imageClient.PullAndArchiveImage(context.TODO(), *request.SourceImage, request.ImageId, request.SourceImageCreds) if err == nil { err = s.imageClient.PullLazy(request) @@ -171,7 +170,7 @@ func (s *Worker) RunContainer(request *types.ContainerRequest) error { if err != nil { return err } - log.Info().Str("container_id", containerId).Int("port", bindPort).Msg("acquired port") + log.Printf("<%s> - acquired port: %d\n", containerId, bindPort) // Read spec from bundle initialBundleSpec, _ := s.readBundleConfig(request.ImageId) @@ -192,7 +191,7 @@ func (s *Worker) RunContainer(request *types.ContainerRequest) error { if err != nil { return err } - log.Info().Str("container_id", containerId).Msg("successfully created spec from request") + log.Printf("<%s> - successfully created spec from request.\n", containerId) // Set an address (ip:port) for the pod/container in Redis. Depending on the stub type, // gateway may need to directly interact with this pod/container. @@ -201,7 +200,7 @@ func (s *Worker) RunContainer(request *types.ContainerRequest) error { if err != nil { return err } - log.Info().Str("container_id", containerId).Msg("set container address") + log.Printf("<%s> - set container address.\n", containerId) logChan := make(chan common.LogRecord) go s.containerWg.Add(1) @@ -209,7 +208,7 @@ func (s *Worker) RunContainer(request *types.ContainerRequest) error { // Start the container go s.spawn(request, spec, logChan, opts) - log.Info().Str("container_id", containerId).Msg("spawned container") + log.Printf("<%s> - spawned successfully.\n", containerId) return nil } @@ -244,6 +243,7 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta spec.Process.Cwd = defaultContainerDirectory spec.Process.Args = request.EntryPoint + spec.Process.Terminal = true // NOTE: This is since we are using a console writer for logging if s.config.Worker.RunCResourcesEnforced { spec.Linux.Resources.CPU = getLinuxCPU(request) @@ -274,6 +274,12 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta spec.Hooks.Prestart = nil } + // We need to modify the spec to support Cedana C/R if enabled + if s.IsCRIUAvailable() && request.CheckpointEnabled { + containerHostname := fmt.Sprintf("%s:%d", s.podAddr, options.BindPort) + s.cedanaClient.PrepareContainerSpec(spec, request.ContainerId, containerHostname, request.RequiresGPU()) + } + spec.Process.Env = append(spec.Process.Env, env...) spec.Root.Readonly = false @@ -292,7 +298,7 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta if _, err := os.Stat(m.LocalPath); os.IsNotExist(err) { err := os.MkdirAll(m.LocalPath, 0755) if err != nil { - log.Error().Str("container_id", request.ContainerId).Err(err).Msg("failed to create mount directory") + log.Printf("<%s> - failed to create mount directory: %v\n", request.ContainerId, err) } } } @@ -305,7 +311,7 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta if m.LinkPath != "" { err = forceSymlink(m.MountPath, m.LinkPath) if err != nil { - log.Error().Str("container_id", request.ContainerId).Err(err).Msg("unable to symlink volume") + log.Printf("<%s> - unable to symlink volume: %v", request.ContainerId, err) } } @@ -321,7 +327,7 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta if s.fileCacheManager.CacheAvailable() && request.Workspace.VolumeCacheEnabled { err = s.fileCacheManager.EnableVolumeCaching(request.Workspace.Name, volumeCacheMap, spec) if err != nil { - log.Error().Str("container_id", request.ContainerId).Err(err).Msg("failed to setup volume caching") + log.Printf("<%s> - failed to setup volume caching: %v", request.ContainerId, err) } } @@ -330,12 +336,14 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta Type: "none", Source: "/workspace/resolv.conf", Destination: "/etc/resolv.conf", - Options: []string{"ro", + Options: []string{ + "ro", "rbind", "rprivate", "nosuid", "noexec", - "nodev"}, + "nodev", + }, } if s.config.Worker.UseHostResolvConf { @@ -343,7 +351,6 @@ func (s *Worker) specFromRequest(request *types.ContainerRequest, options *Conta } spec.Mounts = append(spec.Mounts, resolvMount) - return spec, nil } @@ -360,11 +367,12 @@ func (s *Worker) newSpecTemplate() (*specs.Spec, error) { func (s *Worker) getContainerEnvironment(request *types.ContainerRequest, options *ContainerOptions) []string { // Most of these env vars are required to communicate with the gateway and vice versa env := []string{ - fmt.Sprintf("BIND_PORT=%d", options.BindPort), + fmt.Sprintf("BIND_PORT=%d", containerInnerPort), fmt.Sprintf("CONTAINER_HOSTNAME=%s", fmt.Sprintf("%s:%d", s.podAddr, options.BindPort)), fmt.Sprintf("CONTAINER_ID=%s", request.ContainerId), fmt.Sprintf("BETA9_GATEWAY_HOST=%s", os.Getenv("BETA9_GATEWAY_HOST")), fmt.Sprintf("BETA9_GATEWAY_PORT=%s", os.Getenv("BETA9_GATEWAY_PORT")), + fmt.Sprintf("CHECKPOINT_ENABLED=%t", request.CheckpointEnabled && s.IsCRIUAvailable()), "PYTHONUNBUFFERED=1", } @@ -438,35 +446,39 @@ func (s *Worker) spawn(request *types.ContainerRequest, spec *specs.Spec, logCha if _, err := s.containerRepo.GetContainerState(containerId); err != nil { if _, ok := err.(*types.ErrContainerStateNotFound); ok { - log.Info().Str("container_id", containerId).Msg("container state not found, returning") + log.Printf("<%s> - container state not found, returning\n", containerId) return } } // Update container status to running - s.containerRepo.UpdateContainerStatus(containerId, types.ContainerStatusRunning, time.Duration(types.ContainerStateTtlS)*time.Second) + err := s.containerRepo.UpdateContainerStatus(containerId, types.ContainerStatusRunning, time.Duration(types.ContainerStateTtlS)*time.Second) + if err != nil { + log.Printf("<%s> failed to update container status to running: %v", containerId, err) + } }() // Setup container overlay filesystem err := containerInstance.Overlay.Setup() if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to setup overlay") + log.Printf("<%s> failed to setup overlay: %v", containerId, err) return } defer containerInstance.Overlay.Cleanup() + spec.Root.Path = containerInstance.Overlay.TopLayerPath() // Setup container network namespace / devices err = s.containerNetworkManager.Setup(containerId, spec) if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to setup container network") + log.Printf("<%s> failed to setup container network: %v", containerId, err) return } // Expose the bind port - err = s.containerNetworkManager.ExposePort(containerId, opts.BindPort, opts.BindPort) + err = s.containerNetworkManager.ExposePort(containerId, opts.BindPort, containerInnerPort) if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to expose container bind port") + log.Printf("<%s> failed to expose container bind port: %v", containerId, err) return } @@ -482,34 +494,97 @@ func (s *Worker) spawn(request *types.ContainerRequest, spec *specs.Spec, logCha return } + consoleWriter, err := NewConsoleWriter(containerInstance.OutputWriter) + if err != nil { + log.Printf("<%s> - failed to create console writer: %v\n", containerId, err) + return + } + // Log metrics go s.workerMetrics.EmitContainerUsage(ctx, request) go s.eventRepo.PushContainerStartedEvent(containerId, s.workerId, request) defer func() { go s.eventRepo.PushContainerStoppedEvent(containerId, s.workerId, request) }() - pid := 0 - pidChan := make(chan int, 1) + startedChan := make(chan int, 1) - go func() { - // Wait for runc to start the container - pid = <-pidChan + // Handle checkpoint creation & restore if applicable + if s.IsCRIUAvailable() && request.CheckpointEnabled { + restored, restoredContainerId, err := s.attemptCheckpointOrRestore(ctx, request, consoleWriter, startedChan, configPath) + if err != nil { + log.Printf("<%s> - C/R failed: %v\n", containerId, err) + } - // Capture resource usage (cpu/mem/gpu) - go s.collectAndSendContainerMetrics(ctx, request, spec, pid) + if restored { + // HOTFIX: If we restored from a checkpoint, we need to use the container ID of the restored container + // instead of the original container ID + containerInstance, exists := s.containerInstances.Get(request.ContainerId) + if exists { + containerInstance.Id = restoredContainerId + s.containerInstances.Set(containerId, containerInstance) + containerId = restoredContainerId + } - // Watch for OOM events - go s.watchOOMEvents(ctx, containerId, outputLogger) - }() + exitCode = s.waitForRestoredContainer(ctx, containerId, startedChan, logChan, request, spec) + return + } + } // Invoke runc process (launch the container) - exitCode, err = s.runcHandle.Run(s.ctx, containerId, opts.BundlePath, &runc.CreateOpts{ - OutputWriter: containerInstance.OutputWriter, - ConfigPath: configPath, - Started: pidChan, + _, err = s.runcHandle.Run(s.ctx, containerId, opts.BundlePath, &runc.CreateOpts{ + Detach: true, + ConsoleSocket: consoleWriter, + ConfigPath: configPath, + Started: startedChan, }) + if err != nil { + log.Printf("<%s> - failed to run container: %v\n", containerId, err) + return + } - // Send last log message since the container has exited - outputLogger.Info("", "done", true, "success", err == nil) + exitCode = s.wait(ctx, containerId, startedChan, outputLogger, request, spec) +} + +// Wait for a container to exit and return the exit code +func (s *Worker) wait(ctx context.Context, containerId string, startedChan chan int, outputLogger *slog.Logger, request *types.ContainerRequest, spec *specs.Spec) int { + <-startedChan + + // Clean up runc container state and send final output message + cleanup := func(exitCode int, err error) int { + log.Printf("<%s> - container has exited with code: %d\n", containerId, exitCode) + + outputLogger.Info("", "done", true, "success", exitCode == 0) + + err = s.runcHandle.Delete(s.ctx, containerId, &runc.DeleteOpts{Force: true}) + if err != nil { + log.Printf("<%s> - failed to delete container: %v\n", containerId, err) + } + + return exitCode + } + + // Look up the PID of the container + state, err := s.runcHandle.State(ctx, containerId) + if err != nil { + return cleanup(-1, err) + } + pid := state.Pid + + // Start monitoring the container + go s.collectAndSendContainerMetrics(ctx, request, spec, pid) // Capture resource usage (cpu/mem/gpu) + go s.watchOOMEvents(ctx, containerId, outputLogger) // Watch for OOM events + + process, err := os.FindProcess(pid) + if err != nil { + return cleanup(-1, err) + } + + // Wait for the container to exit + processState, err := process.Wait() + if err != nil { + return cleanup(-1, err) + } + + return cleanup(processState.ExitCode(), nil) } func (s *Worker) createOverlay(request *types.ContainerRequest, bundlePath string) *common.ContainerOverlay { @@ -534,15 +609,24 @@ func (s *Worker) isBuildRequest(request *types.ContainerRequest) bool { } func (s *Worker) watchOOMEvents(ctx context.Context, containerId string, outputLogger *slog.Logger) { - seenEvents := make(map[string]struct{}) + var ( + seenEvents = make(map[string]struct{}) + ch <-chan *runc.Event + err error + ticker = time.NewTicker(time.Second) + ) - ticker := time.NewTicker(time.Second) defer ticker.Stop() - ch, err := s.runcHandle.Events(ctx, containerId, time.Second) - if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to open runc events channel") + select { + case <-ctx.Done(): return + default: + ch, err = s.runcHandle.Events(ctx, containerId, time.Second) + if err != nil { + log.Printf("<%s> failed to open runc events channel: %v", containerId, err) + return + } } maxTries, tries := 5, 0 // Used for re-opening the channel if it's closed @@ -555,7 +639,7 @@ func (s *Worker) watchOOMEvents(ctx context.Context, containerId string, outputL case event, ok := <-ch: if !ok { // If the channel is closed, try to re-open it if tries == maxTries-1 { - log.Error().Str("container_id", containerId).Msg("failed to watch for OOM events") + log.Printf("<%s> failed to watch for OOM events.", containerId) return } @@ -566,7 +650,7 @@ func (s *Worker) watchOOMEvents(ctx context.Context, containerId string, outputL case <-time.After(time.Second): ch, err = s.runcHandle.Events(ctx, containerId, time.Second) if err != nil { - log.Error().Str("container_id", containerId).Err(err).Msg("failed to open runc events channel") + log.Printf("<%s> failed to open runc events channel: %v", containerId, err) } } continue diff --git a/pkg/worker/mount.go b/pkg/worker/mount.go index fa86c0f70..3b4537e60 100644 --- a/pkg/worker/mount.go +++ b/pkg/worker/mount.go @@ -95,3 +95,14 @@ func (c *ContainerMountManager) setupMountPointS3(containerId string, m types.Mo func tempUserCodeDir(containerId string) string { return fmt.Sprintf("/tmp/%s/code", containerId) } + +const ( + checkpointSignalFileName = "READY_FOR_CHECKPOINT" + checkpointCompleteFileName = "CHECKPOINT_COMPLETE" + checkpointContainerIdFileName = "CONTAINER_ID" + checkpointContainerHostnameFileName = "CONTAINER_HOSTNAME" +) + +func checkpointSignalDir(containerId string) string { + return fmt.Sprintf("/tmp/%s/cedana", containerId) +} diff --git a/pkg/worker/runc_console.go b/pkg/worker/runc_console.go new file mode 100644 index 000000000..91ffbaa19 --- /dev/null +++ b/pkg/worker/runc_console.go @@ -0,0 +1,81 @@ +package worker + +import ( + "fmt" + "io" + "net" + "os" + "path/filepath" + "time" + + "github.com/containerd/console" + "github.com/opencontainers/runc/libcontainer/utils" +) + +type ConsoleWriter struct { + path string +} + +// Implementation of runc ConsoleSocket, for writing only +func NewConsoleWriter(writer io.Writer) (*ConsoleWriter, error) { + socketPath := filepath.Join(os.TempDir(), fmt.Sprintf("tty-%d.sock", time.Now().UnixNano())) + + ln, err := net.Listen("unix", socketPath) + if err != nil { + return nil, err + } + + go func() { + // We only accept a single connection, since we can only really have + // one reader for os.Stdin. + conn, err := ln.Accept() + defer ln.Close() + if err != nil { + return + } + defer conn.Close() + + // Close ln, to allow for other instances to take over. + ln.Close() + + // Get the fd of the connection. + unixconn, ok := conn.(*net.UnixConn) + if !ok { + return + } + + socket, err := unixconn.File() + if err != nil { + return + } + defer socket.Close() + + // Get the master file descriptor from runC. + master, err := utils.RecvFd(socket) + if err != nil { + return + } + + c, err := console.ConsoleFromFile(master) + if err != nil { + return + } + defer c.Close() + + if err := console.ClearONLCR(c.Fd()); err != nil { + return + } + + // Copy from our stdio to the master fd. + _, err = io.Copy(writer, c) // Will return on container exit + if err != nil { + return + } + }() + + return &ConsoleWriter{path: socketPath}, nil +} + +func (w *ConsoleWriter) Path() string { + return w.path +} diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index 5720cf2b9..fa4b4e56d 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -26,8 +26,9 @@ const ( requestProcessingInterval time.Duration = 100 * time.Millisecond containerStatusUpdateInterval time.Duration = 30 * time.Second - containerLogsPath string = "/var/log/worker" - defaultWorkerSpindownTimeS float64 = 300 // 5 minutes + containerLogsPath string = "/var/log/worker" + defaultWorkerSpindownTimeS float64 = 300 // 5 minutes + defaultCacheWaitTime time.Duration = 30 * time.Second ) type Worker struct { @@ -40,6 +41,7 @@ type Worker struct { imageMountPath string runcHandle runc.Runc runcServer *RunCServer + cedanaClient *CedanaClient fileCacheManager *FileCacheManager containerNetworkManager *ContainerNetworkManager containerCudaManager GPUManager @@ -58,7 +60,8 @@ type Worker struct { stopContainerChan chan stopContainerEvent workerRepo repo.WorkerRepository eventRepo repo.EventRepository - storage storage.Storage + userDataStorage storage.Storage + checkpointStorage storage.Storage ctx context.Context cancel func() config types.AppConfig @@ -94,6 +97,7 @@ func NewWorker() (*Worker, error) { gpuType := os.Getenv("GPU_TYPE") workerId := os.Getenv("WORKER_ID") + workerPoolName := os.Getenv("WORKER_POOL_NAME") podHostName := os.Getenv("HOSTNAME") podAddr, err := GetPodAddr() @@ -131,7 +135,7 @@ func NewWorker() (*Worker, error) { workerRepo := repo.NewWorkerRedisRepository(redisClient, config.Worker) eventRepo := repo.NewTCPEventClientRepo(config.Monitoring.FluentBit.Events) - storage, err := storage.NewStorage(config.Storage) + userDataStorage, err := storage.NewStorage(config.Storage) if err != nil { return nil, err } @@ -139,6 +143,10 @@ func NewWorker() (*Worker, error) { var cacheClient *blobcache.BlobCacheClient = nil if config.Worker.BlobCacheEnabled { cacheClient, err = blobcache.NewBlobCacheClient(context.TODO(), config.BlobCache) + if err == nil { + err = cacheClient.WaitForHosts(defaultCacheWaitTime) + } + if err != nil { log.Warn().Err(err).Msg("cache unavailable, performance may be degraded") } @@ -150,6 +158,35 @@ func NewWorker() (*Worker, error) { return nil, err } + var cedanaClient *CedanaClient = nil + var checkpointStorage storage.Storage = nil + if pool, ok := config.Worker.Pools[workerPoolName]; ok && pool.CRIUEnabled { + cedanaClient, err = NewCedanaClient(context.Background(), config.Worker.CRIU.Cedana, gpuType != "") + if err != nil { + log.Printf("[WARNING] C/R unavailable, failed to create cedana client: %v\n", err) + } + + os.MkdirAll(config.Worker.CRIU.Storage.MountPath, os.ModePerm) + + // If storage mode is S3, mount the checkpoint storage as a FUSE filesystem + if config.Worker.CRIU.Storage.Mode == string(types.CheckpointStorageModeS3) { + checkpointStorage, _ := storage.NewMountPointStorage(types.MountPointConfig{ + S3Bucket: config.Worker.CRIU.Storage.ObjectStore.BucketName, + AccessKey: config.Worker.CRIU.Storage.ObjectStore.AccessKey, + SecretKey: config.Worker.CRIU.Storage.ObjectStore.SecretKey, + EndpointURL: config.Worker.CRIU.Storage.ObjectStore.EndpointURL, + Region: config.Worker.CRIU.Storage.ObjectStore.Region, + ReadOnly: false, + }) + + err := checkpointStorage.Mount(config.Worker.CRIU.Storage.MountPath) + if err != nil { + log.Printf("[WARNING] C/R unavailable, unable to mount checkpoint storage: %v\n", err) + cedanaClient = nil + } + } + } + runcServer, err := NewRunCServer(containerInstances, imageClient) if err != nil { return nil, err @@ -191,6 +228,7 @@ func NewWorker() (*Worker, error) { containerMountManager: NewContainerMountManager(config), podAddr: podAddr, imageClient: imageClient, + cedanaClient: cedanaClient, podHostName: podHostName, eventBus: nil, workerId: workerId, @@ -206,7 +244,8 @@ func NewWorker() (*Worker, error) { eventRepo: eventRepo, completedRequests: make(chan *types.ContainerRequest, 1000), stopContainerChan: make(chan stopContainerEvent, 1000), - storage: storage, + userDataStorage: userDataStorage, + checkpointStorage: checkpointStorage, }, nil } @@ -325,6 +364,12 @@ func (s *Worker) updateContainerStatus(request *types.ContainerRequest) error { log.Info().Str("container_id", request.ContainerId).Str("image_id", request.ImageId).Msg("container still running") + // TODO: remove this hotfix + if state.Status == types.ContainerStatusPending { + log.Printf("<%s> - forcing container status to running\n", request.ContainerId) + state.Status = types.ContainerStatusRunning + } + err = s.containerRepo.UpdateContainerStatus(request.ContainerId, state.Status, time.Duration(types.ContainerStateTtlS)*time.Second) if err != nil { log.Error().Str("container_id", request.ContainerId).Err(err).Msg("unable to update container state") @@ -448,9 +493,16 @@ func (s *Worker) shutdown() error { } } - err := s.storage.Unmount(s.config.Storage.FilesystemPath) + err := s.userDataStorage.Unmount(s.config.Storage.FilesystemPath) if err != nil { - errs = errors.Join(errs, fmt.Errorf("failed to unmount storage: %v", err)) + errs = errors.Join(errs, fmt.Errorf("failed to unmount data storage: %v", err)) + } + + if s.checkpointStorage != nil { + err = s.checkpointStorage.Unmount(s.config.Worker.CRIU.Storage.MountPath) + if err != nil { + errs = errors.Join(errs, fmt.Errorf("failed to unmount checkpoint storage: %v", err)) + } } err = s.imageClient.Cleanup() diff --git a/proto/gateway.pb.go b/proto/gateway.pb.go index 05c51472e..06427bcb3 100644 --- a/proto/gateway.pb.go +++ b/proto/gateway.pb.go @@ -2035,6 +2035,8 @@ type GetOrCreateStubRequest struct { TaskPolicy *TaskPolicy `protobuf:"bytes,23,opt,name=task_policy,json=taskPolicy,proto3" json:"task_policy,omitempty"` ConcurrentRequests uint32 `protobuf:"varint,24,opt,name=concurrent_requests,json=concurrentRequests,proto3" json:"concurrent_requests,omitempty"` Extra string `protobuf:"bytes,25,opt,name=extra,proto3" json:"extra,omitempty"` + CheckpointEnabled bool `protobuf:"varint,26,opt,name=checkpoint_enabled,json=checkpointEnabled,proto3" json:"checkpoint_enabled,omitempty"` + GpuCount uint32 `protobuf:"varint,27,opt,name=gpu_count,json=gpuCount,proto3" json:"gpu_count,omitempty"` } func (x *GetOrCreateStubRequest) Reset() { @@ -2237,6 +2239,20 @@ func (x *GetOrCreateStubRequest) GetExtra() string { return "" } +func (x *GetOrCreateStubRequest) GetCheckpointEnabled() bool { + if x != nil { + return x.CheckpointEnabled + } + return false +} + +func (x *GetOrCreateStubRequest) GetGpuCount() uint32 { + if x != nil { + return x.GpuCount + } + return 0 +} + type GetOrCreateStubResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -5180,7 +5196,7 @@ var file_gateway_proto_rawDesc = []byte{ 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x72, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x52, 0x65, 0x74, 0x72, 0x69, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x74, - 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x22, 0xb0, 0x06, 0x0a, + 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x74, 0x74, 0x6c, 0x22, 0xfc, 0x06, 0x0a, 0x16, 0x47, 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6f, 0x62, 0x6a, 0x65, @@ -5231,499 +5247,503 @@ var file_gateway_proto_rawDesc = []byte{ 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x12, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x78, 0x74, - 0x72, 0x61, 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x22, - 0x76, 0x0a, 0x17, 0x47, 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, - 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, - 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, - 0x62, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x19, 0x0a, 0x08, - 0x77, 0x61, 0x72, 0x6e, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x77, 0x61, 0x72, 0x6e, 0x4d, 0x73, 0x67, 0x22, 0x40, 0x0a, 0x11, 0x44, 0x65, 0x70, 0x6c, 0x6f, - 0x79, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, - 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, - 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, - 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, - 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, - 0x1d, 0x0a, 0x0a, 0x69, 0x6e, 0x76, 0x6f, 0x6b, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6e, 0x76, 0x6f, 0x6b, 0x65, 0x55, 0x72, 0x6c, 0x22, 0xf5, - 0x02, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, - 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, - 0x62, 0x5f, 0x69, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, - 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x73, 0x74, 0x75, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x1b, 0x0a, 0x09, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x08, 0x73, 0x74, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, - 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x77, 0x6f, - 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x6f, 0x72, - 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0d, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, - 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x0a, + 0x72, 0x61, 0x18, 0x19, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x78, 0x74, 0x72, 0x61, 0x12, + 0x2d, 0x0a, 0x12, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x65, 0x6e, + 0x61, 0x62, 0x6c, 0x65, 0x64, 0x18, 0x1a, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x63, 0x68, 0x65, + 0x63, 0x6b, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x67, 0x70, 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x1b, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x08, 0x67, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x76, 0x0a, 0x17, 0x47, + 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, + 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x77, 0x61, 0x72, 0x6e, + 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x77, 0x61, 0x72, 0x6e, + 0x4d, 0x73, 0x67, 0x22, 0x40, 0x0a, 0x11, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x53, 0x74, 0x75, + 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, + 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x82, 0x01, 0x0a, 0x12, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, + 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x23, 0x0a, 0x0d, + 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x49, + 0x64, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x69, + 0x6e, 0x76, 0x6f, 0x6b, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x69, 0x6e, 0x76, 0x6f, 0x6b, 0x65, 0x55, 0x72, 0x6c, 0x22, 0xf5, 0x02, 0x0a, 0x0a, 0x44, + 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, + 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x1b, + 0x0a, 0x09, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x08, 0x73, 0x74, 0x75, 0x62, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x73, + 0x74, 0x75, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x73, 0x74, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, + 0x69, 0x6f, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x21, 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, + 0x69, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x49, 0x64, 0x12, 0x25, 0x0a, 0x0e, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x77, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x39, 0x0a, 0x0a, + 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, + 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, + 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, + 0x41, 0x74, 0x22, 0xc7, 0x01, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, + 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x46, 0x0a, + 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, + 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, + 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, + 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x66, 0x69, + 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x1a, 0x4f, 0x0a, 0x0c, 0x46, + 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x67, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, + 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x79, 0x0a, 0x17, + 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, + 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, + 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, + 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, 0x64, 0x65, 0x70, 0x6c, + 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x27, 0x0a, 0x15, 0x53, 0x74, 0x6f, 0x70, 0x44, + 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, + 0x22, 0x41, 0x0a, 0x16, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, + 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, + 0x4d, 0x73, 0x67, 0x22, 0x29, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, + 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x22, 0x43, + 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, + 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, + 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, + 0x4d, 0x73, 0x67, 0x22, 0xbe, 0x02, 0x0a, 0x04, 0x50, 0x6f, 0x6f, 0x6c, 0x12, 0x12, 0x0a, 0x04, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, + 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, 0x75, 0x18, + 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x69, + 0x6e, 0x46, 0x72, 0x65, 0x65, 0x47, 0x70, 0x75, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x47, 0x70, 0x75, 0x12, 0x1e, 0x0a, 0x0a, 0x6d, 0x69, + 0x6e, 0x46, 0x72, 0x65, 0x65, 0x43, 0x70, 0x75, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x43, 0x70, 0x75, 0x12, 0x24, 0x0a, 0x0d, 0x6d, 0x69, + 0x6e, 0x46, 0x72, 0x65, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x0d, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, + 0x12, 0x2a, 0x0a, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x43, 0x70, 0x75, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x64, 0x65, 0x66, 0x61, + 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x43, 0x70, 0x75, 0x12, 0x30, 0x0a, 0x13, + 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x65, 0x6d, + 0x6f, 0x72, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, 0x65, 0x66, 0x61, 0x75, + 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x34, + 0x0a, 0x15, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x47, + 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x15, 0x64, + 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x47, 0x70, 0x75, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xbb, 0x01, 0x0a, 0x10, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, + 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, 0x07, 0x66, 0x69, 0x6c, + 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, + 0x72, 0x79, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x1a, 0x4f, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, + 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x72, + 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x61, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, + 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, + 0x12, 0x23, 0x0a, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x0d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, 0x6f, 0x6f, 0x6c, 0x52, 0x05, + 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x22, 0x8c, 0x04, 0x0a, 0x07, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, + 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, + 0x63, 0x70, 0x75, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x67, + 0x70, 0x75, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x1b, 0x0a, + 0x09, 0x67, 0x70, 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, + 0x52, 0x08, 0x67, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, + 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, + 0x23, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, + 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x11, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, + 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x25, 0x0a, 0x0e, 0x74, 0x61, 0x69, 0x6c, + 0x73, 0x63, 0x61, 0x6c, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0d, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x41, 0x75, 0x74, 0x68, 0x12, + 0x25, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, + 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x61, 0x73, 0x74, 0x4b, 0x65, 0x65, + 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, + 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x56, 0x65, + 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x0f, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x0e, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x72, 0x5f, + 0x64, 0x61, 0x74, 0x61, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, + 0x44, 0x61, 0x74, 0x61, 0x22, 0xf3, 0x03, 0x0a, 0x0e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x70, 0x75, 0x41, 0x76, + 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x2e, 0x0a, + 0x13, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x5f, 0x70, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, 0x63, 0x70, 0x75, 0x55, + 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x63, 0x74, 0x12, 0x34, 0x0a, + 0x16, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x02, 0x52, 0x14, 0x6d, + 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x50, 0x63, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, + 0x6e, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, + 0x24, 0x0a, 0x0e, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x67, 0x70, 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, + 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x66, 0x72, 0x65, 0x65, 0x47, 0x70, 0x75, + 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x75, + 0x73, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x63, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0d, + 0x63, 0x61, 0x63, 0x68, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x50, 0x63, 0x74, 0x12, 0x25, 0x0a, + 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x43, 0x61, 0x70, 0x61, + 0x63, 0x69, 0x74, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x6d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x10, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x55, 0x73, 0x61, + 0x67, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x63, 0x70, 0x75, 0x5f, + 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0d, 0x63, 0x61, 0x63, + 0x68, 0x65, 0x43, 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x22, 0x48, 0x0a, 0x13, 0x4c, 0x69, + 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x14, + 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, + 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, + 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x2c, 0x0a, 0x08, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, + 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x08, 0x6d, 0x61, 0x63, 0x68, + 0x69, 0x6e, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x04, 0x67, 0x70, 0x75, 0x73, 0x18, 0x04, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, + 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2e, 0x47, 0x70, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x04, 0x67, 0x70, 0x75, + 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x47, 0x70, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x33, 0x0a, 0x14, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x22, + 0x6c, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, + 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, + 0x67, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x61, 0x63, + 0x68, 0x69, 0x6e, 0x65, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x22, 0x52, 0x0a, + 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, 0x6d, + 0x65, 0x22, 0x40, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, + 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, + 0x4d, 0x73, 0x67, 0x22, 0xb6, 0x02, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x19, 0x0a, + 0x08, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x07, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, + 0x76, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x75, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x75, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x26, + 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, + 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x74, 0x6f, 0x6b, 0x65, + 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, + 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, + 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, - 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, - 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x22, 0xc7, 0x01, 0x0a, 0x16, 0x4c, 0x69, 0x73, 0x74, 0x44, - 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x46, 0x0a, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, - 0x52, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, - 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x1a, - 0x4f, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, - 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x13, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, - 0x22, 0x79, 0x0a, 0x17, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, - 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, - 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, - 0x72, 0x4d, 0x73, 0x67, 0x12, 0x35, 0x0a, 0x0b, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, - 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x67, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x0b, - 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0x27, 0x0a, 0x15, 0x53, - 0x74, 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x02, 0x69, 0x64, 0x22, 0x41, 0x0a, 0x16, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, - 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, - 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, - 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, 0x29, 0x0a, 0x17, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x22, 0x43, 0x0a, 0x18, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, - 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, - 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, - 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, 0xbe, 0x02, 0x0a, 0x04, 0x50, 0x6f, 0x6f, 0x6c, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x10, 0x0a, 0x03, - 0x67, 0x70, 0x75, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x1e, - 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x47, 0x70, 0x75, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x47, 0x70, 0x75, 0x12, 0x1e, - 0x0a, 0x0a, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x43, 0x70, 0x75, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x43, 0x70, 0x75, 0x12, 0x24, - 0x0a, 0x0d, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, - 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6d, 0x69, 0x6e, 0x46, 0x72, 0x65, 0x65, 0x4d, 0x65, - 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x2a, 0x0a, 0x10, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x43, 0x70, 0x75, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, - 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x43, 0x70, 0x75, - 0x12, 0x30, 0x0a, 0x13, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x09, 0x52, 0x13, 0x64, - 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x4d, 0x65, 0x6d, 0x6f, - 0x72, 0x79, 0x12, 0x34, 0x0a, 0x15, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, - 0x6b, 0x65, 0x72, 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0a, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x15, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, - 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xbb, 0x01, 0x0a, 0x10, 0x4c, 0x69, 0x73, - 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x40, 0x0a, - 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, - 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, - 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x07, 0x66, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, 0x12, - 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, - 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x1a, 0x4f, 0x0a, 0x0c, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x73, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x61, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, - 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, - 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, - 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, - 0x72, 0x4d, 0x73, 0x67, 0x12, 0x23, 0x0a, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, 0x6f, - 0x6f, 0x6c, 0x52, 0x05, 0x70, 0x6f, 0x6f, 0x6c, 0x73, 0x22, 0x8c, 0x04, 0x0a, 0x07, 0x4d, 0x61, - 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x63, 0x70, 0x75, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x03, 0x52, 0x03, 0x63, 0x70, 0x75, 0x12, 0x16, 0x0a, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, - 0x10, 0x0a, 0x03, 0x67, 0x70, 0x75, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, - 0x75, 0x12, 0x1b, 0x0a, 0x09, 0x67, 0x70, 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x0d, 0x52, 0x08, 0x67, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x72, 0x6f, 0x76, 0x69, 0x64, 0x65, 0x72, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x72, 0x6f, 0x76, - 0x69, 0x64, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2d, 0x0a, 0x12, 0x72, 0x65, 0x67, 0x69, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x11, 0x72, 0x65, 0x67, 0x69, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x23, 0x0a, 0x0d, 0x74, 0x61, 0x69, 0x6c, 0x73, - 0x63, 0x61, 0x6c, 0x65, 0x5f, 0x75, 0x72, 0x6c, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, - 0x74, 0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x55, 0x72, 0x6c, 0x12, 0x25, 0x0a, 0x0e, - 0x74, 0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x5f, 0x61, 0x75, 0x74, 0x68, 0x18, 0x0b, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x63, 0x61, 0x6c, 0x65, 0x41, - 0x75, 0x74, 0x68, 0x12, 0x25, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, 0x5f, 0x6b, 0x65, 0x65, 0x70, - 0x61, 0x6c, 0x69, 0x76, 0x65, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x6c, 0x61, 0x73, - 0x74, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x63, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x61, 0x67, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x61, 0x67, 0x65, - 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x40, 0x0a, 0x0f, 0x6d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x18, 0x0f, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x52, 0x0e, 0x6d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x1b, 0x0a, 0x09, 0x75, - 0x73, 0x65, 0x72, 0x5f, 0x64, 0x61, 0x74, 0x61, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x75, 0x73, 0x65, 0x72, 0x44, 0x61, 0x74, 0x61, 0x22, 0xf3, 0x03, 0x0a, 0x0e, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x12, 0x2e, 0x0a, 0x13, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x70, 0x75, 0x5f, 0x61, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, - 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x11, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, - 0x70, 0x75, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x61, 0x76, 0x61, 0x69, - 0x6c, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x74, 0x6f, 0x74, - 0x61, 0x6c, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x6c, - 0x65, 0x12, 0x2e, 0x0a, 0x13, 0x63, 0x70, 0x75, 0x5f, 0x75, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, - 0x63, 0x70, 0x75, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x63, - 0x74, 0x12, 0x34, 0x0a, 0x16, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x74, 0x69, 0x6c, - 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x63, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x02, 0x52, 0x14, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x55, 0x74, 0x69, 0x6c, 0x69, 0x7a, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x63, 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0b, 0x77, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x27, 0x0a, 0x0f, 0x63, 0x6f, - 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x24, 0x0a, 0x0e, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x67, 0x70, 0x75, 0x5f, - 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0c, 0x66, 0x72, 0x65, - 0x65, 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x63, - 0x68, 0x65, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x63, 0x74, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x02, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x55, 0x73, 0x61, 0x67, 0x65, 0x50, 0x63, - 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, 0x63, 0x61, 0x70, 0x61, 0x63, - 0x69, 0x74, 0x79, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, - 0x43, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, 0x79, 0x12, 0x2c, 0x0a, 0x12, 0x63, 0x61, 0x63, 0x68, - 0x65, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0a, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x63, 0x61, 0x63, 0x68, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x55, 0x73, 0x61, 0x67, 0x65, 0x12, 0x26, 0x0a, 0x0f, 0x63, 0x61, 0x63, 0x68, 0x65, 0x5f, - 0x63, 0x70, 0x75, 0x5f, 0x75, 0x73, 0x61, 0x67, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x02, 0x52, - 0x0d, 0x63, 0x61, 0x63, 0x68, 0x65, 0x43, 0x70, 0x75, 0x55, 0x73, 0x61, 0x67, 0x65, 0x22, 0x48, - 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, - 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x22, 0xe3, 0x01, 0x0a, 0x14, 0x4c, 0x69, 0x73, - 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, - 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x2c, 0x0a, 0x08, 0x6d, 0x61, - 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x08, - 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x3b, 0x0a, 0x04, 0x67, 0x70, 0x75, 0x73, - 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x47, 0x70, 0x75, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, - 0x04, 0x67, 0x70, 0x75, 0x73, 0x1a, 0x37, 0x0a, 0x09, 0x47, 0x70, 0x75, 0x73, 0x45, 0x6e, 0x74, - 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x33, - 0x0a, 0x14, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, - 0x61, 0x6d, 0x65, 0x22, 0x6c, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, - 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, - 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, - 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x2a, 0x0a, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x2e, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x07, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, - 0x65, 0x22, 0x52, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, - 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, - 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x40, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, - 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, - 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, 0xb6, 0x02, 0x0a, 0x05, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x12, 0x10, 0x0a, 0x03, - 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, - 0x0a, 0x06, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x72, 0x65, 0x75, 0x73, 0x61, 0x62, - 0x6c, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x72, 0x65, 0x75, 0x73, 0x61, 0x62, - 0x6c, 0x65, 0x12, 0x26, 0x0a, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, - 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x48, 0x00, 0x52, 0x0b, 0x77, 0x6f, 0x72, 0x6b, - 0x73, 0x70, 0x61, 0x63, 0x65, 0x49, 0x64, 0x88, 0x01, 0x01, 0x12, 0x1d, 0x0a, 0x0a, 0x74, 0x6f, - 0x6b, 0x65, 0x6e, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x12, 0x39, 0x0a, 0x0a, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x5f, 0x61, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x63, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x64, 0x41, 0x74, 0x12, 0x39, 0x0a, 0x0a, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x5f, - 0x61, 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x42, - 0x0f, 0x0a, 0x0d, 0x5f, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, - 0x22, 0x13, 0x0a, 0x11, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x65, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, - 0x65, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, - 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, - 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, - 0x72, 0x4d, 0x73, 0x67, 0x12, 0x26, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x22, 0x14, 0x0a, 0x12, - 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x22, 0x64, 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, - 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, - 0x73, 0x67, 0x12, 0x24, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x2f, 0x0a, 0x12, 0x54, 0x6f, 0x67, 0x67, - 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, - 0x0a, 0x08, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x22, 0x64, 0x0a, 0x13, 0x54, 0x6f, 0x67, - 0x67, 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, - 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x24, 0x0a, 0x05, 0x74, 0x6f, 0x6b, - 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, - 0x2f, 0x0a, 0x12, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, - 0x22, 0x3e, 0x0a, 0x13, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, + 0x52, 0x09, 0x75, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74, 0x42, 0x0f, 0x0a, 0x0d, 0x5f, + 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x22, 0x13, 0x0a, 0x11, + 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x22, 0x65, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, - 0x22, 0x68, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x0c, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, - 0x19, 0x0a, 0x08, 0x75, 0x72, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x07, 0x75, 0x72, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x22, 0x4b, 0x0a, 0x0e, 0x47, 0x65, - 0x74, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, + 0x12, 0x26, 0x0a, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x52, 0x06, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x22, 0x14, 0x0a, 0x12, 0x43, 0x72, 0x65, 0x61, + 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x64, + 0x0a, 0x13, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x24, + 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x05, 0x74, + 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x2f, 0x0a, 0x12, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6f, + 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x22, 0x64, 0x0a, 0x13, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, - 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x03, 0x75, 0x72, 0x6c, 0x22, 0xca, 0x03, 0x0a, 0x06, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, - 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, - 0x75, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x1b, 0x0a, 0x09, - 0x70, 0x6f, 0x6f, 0x6c, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x70, 0x6f, 0x6f, 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x63, - 0x68, 0x69, 0x6e, 0x65, 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, - 0x72, 0x69, 0x74, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x72, 0x69, 0x6f, - 0x72, 0x69, 0x74, 0x79, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x70, - 0x75, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x08, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x70, - 0x75, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, - 0x79, 0x18, 0x08, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, - 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x67, 0x70, - 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, - 0x6f, 0x74, 0x61, 0x6c, 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, - 0x66, 0x72, 0x65, 0x65, 0x5f, 0x63, 0x70, 0x75, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, - 0x66, 0x72, 0x65, 0x65, 0x43, 0x70, 0x75, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x72, 0x65, 0x65, 0x5f, - 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, 0x72, - 0x65, 0x65, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x0e, 0x66, 0x72, 0x65, 0x65, - 0x5f, 0x67, 0x70, 0x75, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, - 0x52, 0x0c, 0x66, 0x72, 0x65, 0x65, 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3f, - 0x0a, 0x11, 0x61, 0x63, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, - 0x65, 0x72, 0x73, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x10, 0x61, - 0x63, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x12, - 0x23, 0x0a, 0x0d, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x0e, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x14, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x69, 0x0a, 0x13, 0x4c, 0x69, - 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, - 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x29, 0x0a, 0x07, 0x77, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x67, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x07, 0x77, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x73, 0x22, 0x32, 0x0a, 0x13, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, - 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x22, 0x3f, 0x0a, 0x14, 0x43, 0x6f, 0x72, - 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, - 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, 0x34, 0x0a, 0x15, 0x55, 0x6e, - 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, - 0x22, 0x41, 0x0a, 0x16, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, - 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, - 0x4d, 0x73, 0x67, 0x22, 0x31, 0x0a, 0x12, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, - 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x22, 0x3e, 0x0a, 0x13, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, + 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x24, 0x0a, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x05, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x22, 0x2f, 0x0a, 0x12, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x49, 0x64, 0x22, 0x3e, 0x0a, 0x13, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, + 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, 0x68, 0x0a, 0x0d, + 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, + 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x70, 0x6c, 0x6f, 0x79, + 0x6d, 0x65, 0x6e, 0x74, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, + 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x19, 0x0a, 0x08, 0x75, + 0x72, 0x6c, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x75, + 0x72, 0x6c, 0x54, 0x79, 0x70, 0x65, 0x22, 0x4b, 0x0a, 0x0e, 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, + 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, + 0x67, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x72, 0x6c, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, + 0x75, 0x72, 0x6c, 0x22, 0xca, 0x03, 0x0a, 0x06, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x0e, + 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x16, + 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x67, 0x70, 0x75, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x03, 0x67, 0x70, 0x75, 0x12, 0x1b, 0x0a, 0x09, 0x70, 0x6f, 0x6f, 0x6c, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x6f, 0x6f, + 0x6c, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x6d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, + 0x5f, 0x69, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x49, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x70, 0x72, 0x69, 0x6f, 0x72, 0x69, 0x74, 0x79, + 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x63, 0x70, 0x75, 0x18, 0x07, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x08, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x70, 0x75, 0x12, 0x21, 0x0a, + 0x0c, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, 0x72, 0x79, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79, + 0x12, 0x26, 0x0a, 0x0f, 0x74, 0x6f, 0x74, 0x61, 0x6c, 0x5f, 0x67, 0x70, 0x75, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0d, 0x74, 0x6f, 0x74, 0x61, 0x6c, + 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x66, 0x72, 0x65, 0x65, + 0x5f, 0x63, 0x70, 0x75, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x03, 0x52, 0x07, 0x66, 0x72, 0x65, 0x65, + 0x43, 0x70, 0x75, 0x12, 0x1f, 0x0a, 0x0b, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x6d, 0x65, 0x6d, 0x6f, + 0x72, 0x79, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x66, 0x72, 0x65, 0x65, 0x4d, 0x65, + 0x6d, 0x6f, 0x72, 0x79, 0x12, 0x24, 0x0a, 0x0e, 0x66, 0x72, 0x65, 0x65, 0x5f, 0x67, 0x70, 0x75, + 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0c, 0x66, 0x72, + 0x65, 0x65, 0x47, 0x70, 0x75, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x3f, 0x0a, 0x11, 0x61, 0x63, + 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x18, + 0x0d, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, + 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x10, 0x61, 0x63, 0x74, 0x69, 0x76, + 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x62, + 0x75, 0x69, 0x6c, 0x64, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x62, 0x75, 0x69, 0x6c, 0x64, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, + 0x22, 0x14, 0x0a, 0x12, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x69, 0x0a, 0x13, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, + 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, + 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, + 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, + 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x29, 0x0a, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x07, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x73, 0x22, 0x32, 0x0a, 0x13, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, + 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x49, 0x64, 0x22, 0x3f, 0x0a, 0x14, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x2a, 0x41, 0x0a, 0x1d, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, - 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4f, 0x70, - 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x09, 0x0a, 0x05, 0x57, 0x52, 0x49, 0x54, 0x45, - 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x01, 0x12, 0x09, - 0x0a, 0x05, 0x4d, 0x4f, 0x56, 0x45, 0x44, 0x10, 0x02, 0x32, 0xfb, 0x11, 0x0a, 0x0e, 0x47, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x09, - 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x41, - 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x4a, 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, - 0x64, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, - 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x50, 0x61, 0x79, - 0x6c, 0x6f, 0x61, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x47, - 0x0a, 0x0a, 0x48, 0x65, 0x61, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1a, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x09, 0x50, 0x75, 0x74, 0x4f, 0x62, - 0x6a, 0x65, 0x63, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, - 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, - 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, - 0x0f, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, 0x34, 0x0a, 0x15, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, + 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x49, 0x64, 0x22, 0x41, 0x0a, 0x16, + 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, 0x5f, 0x6d, 0x73, + 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, 0x73, 0x67, 0x22, + 0x31, 0x0a, 0x12, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x77, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x49, 0x64, 0x22, 0x3e, 0x0a, 0x13, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x17, 0x0a, 0x07, 0x65, 0x72, 0x72, + 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x65, 0x72, 0x72, 0x4d, + 0x73, 0x67, 0x2a, 0x41, 0x0a, 0x1d, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x62, 0x6a, + 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x09, 0x0a, 0x05, 0x57, 0x52, 0x49, 0x54, 0x45, 0x10, 0x00, 0x12, 0x0a, + 0x0a, 0x06, 0x44, 0x45, 0x4c, 0x45, 0x54, 0x45, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x4d, 0x4f, + 0x56, 0x45, 0x44, 0x10, 0x02, 0x32, 0xfb, 0x11, 0x0a, 0x0e, 0x47, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x44, 0x0a, 0x09, 0x41, 0x75, 0x74, 0x68, + 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, + 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x41, 0x75, 0x74, 0x68, 0x6f, + 0x72, 0x69, 0x7a, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4a, + 0x0a, 0x0b, 0x53, 0x69, 0x67, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, 0x12, 0x1b, 0x2e, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x50, 0x61, 0x79, 0x6c, + 0x6f, 0x61, 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x50, 0x61, 0x79, 0x6c, 0x6f, 0x61, 0x64, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x47, 0x0a, 0x0a, 0x48, 0x65, + 0x61, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x48, + 0x65, 0x61, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x44, 0x0a, 0x09, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x65, 0x0a, 0x14, 0x52, - 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x6e, 0x74, 0x12, 0x24, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x52, 0x65, - 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, - 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x00, 0x12, 0x53, 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, - 0x6e, 0x65, 0x72, 0x73, 0x12, 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, - 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x0d, 0x53, 0x74, 0x6f, 0x70, 0x43, - 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, - 0x07, 0x45, 0x6e, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x17, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x45, 0x6e, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x18, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x45, 0x6e, 0x64, 0x54, - 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x53, - 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, - 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x42, 0x0a, 0x09, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x19, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x73, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x4c, 0x0a, 0x0f, 0x50, 0x75, 0x74, + 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x19, 0x2e, 0x67, + 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x53, 0x74, 0x75, 0x62, 0x12, 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, - 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x75, 0x62, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x2e, 0x47, 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x75, - 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x44, 0x65, 0x70, - 0x6c, 0x6f, 0x79, 0x53, 0x74, 0x75, 0x62, 0x12, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, - 0x79, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x39, 0x0a, 0x06, 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x12, 0x16, 0x2e, 0x67, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, - 0x55, 0x52, 0x4c, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x4c, - 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1f, - 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, - 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x20, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, - 0x65, 0x6e, 0x74, 0x12, 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, - 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, - 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x57, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, - 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x20, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, - 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x67, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, - 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, - 0x09, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, - 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, - 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, - 0x73, 0x12, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, - 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, - 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, - 0x1d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, - 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, - 0x1d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, - 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, - 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, - 0x0a, 0x0a, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, + 0x79, 0x2e, 0x50, 0x75, 0x74, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01, 0x12, 0x65, 0x0a, 0x14, 0x52, 0x65, 0x70, 0x6c, 0x61, + 0x63, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, + 0x24, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, + 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, + 0x52, 0x65, 0x70, 0x6c, 0x61, 0x63, 0x65, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x43, 0x6f, 0x6e, + 0x74, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x53, + 0x0a, 0x0e, 0x4c, 0x69, 0x73, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, + 0x12, 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x43, + 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x22, 0x00, 0x12, 0x50, 0x0a, 0x0d, 0x53, 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6e, 0x74, 0x61, + 0x69, 0x6e, 0x65, 0x72, 0x12, 0x1d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, + 0x74, 0x6f, 0x70, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, + 0x6f, 0x70, 0x43, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x42, 0x0a, 0x09, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, + 0x73, 0x6b, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, + 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, + 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x07, 0x45, 0x6e, 0x64, + 0x54, 0x61, 0x73, 0x6b, 0x12, 0x17, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x45, + 0x6e, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x45, 0x6e, 0x64, 0x54, 0x61, 0x73, 0x6b, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x53, 0x74, 0x6f, 0x70, 0x54, + 0x61, 0x73, 0x6b, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, + 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, + 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x4c, + 0x69, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, + 0x73, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, + 0x54, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, + 0x75, 0x62, 0x12, 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, + 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x65, + 0x74, 0x4f, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x53, + 0x74, 0x75, 0x62, 0x12, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, + 0x53, 0x74, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x39, 0x0a, 0x06, + 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x12, 0x16, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x2e, 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, + 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x47, 0x65, 0x74, 0x55, 0x52, 0x4c, 0x52, + 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x54, 0x0a, 0x0f, 0x4c, 0x69, 0x73, 0x74, 0x44, + 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1f, 0x2e, 0x67, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, + 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x67, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, + 0x0e, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x12, + 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x65, + 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x57, 0x0a, 0x10, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, + 0x6d, 0x65, 0x6e, 0x74, 0x12, 0x20, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x21, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, + 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x09, 0x4c, 0x69, 0x73, + 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x12, 0x19, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, + 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, + 0x50, 0x6f, 0x6f, 0x6c, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, + 0x0c, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x73, 0x12, 0x1c, 0x2e, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, + 0x69, 0x6e, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x67, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, + 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x43, 0x72, + 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x2e, 0x67, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, + 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4e, 0x0a, 0x0d, 0x44, 0x65, + 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, 0x6e, 0x65, 0x12, 0x1d, 0x2e, 0x67, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, + 0x69, 0x6e, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1e, 0x2e, 0x67, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x63, 0x68, 0x69, + 0x6e, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x45, 0x0a, 0x0a, 0x4c, 0x69, + 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x12, 0x1a, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, + 0x69, 0x73, 0x74, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, + 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, + 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x6f, + 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x54, + 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, + 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x2e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, + 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, + 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x48, 0x0a, 0x0b, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, - 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x54, 0x6f, 0x67, 0x67, 0x6c, 0x65, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x65, 0x6c, - 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, - 0x61, 0x79, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, + 0x48, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x1b, + 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, + 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, + 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, 0x0c, 0x43, 0x6f, 0x72, + 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, + 0x77, 0x61, 0x79, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, + 0x79, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, + 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x1e, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, + 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x44, 0x72, 0x61, + 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, + 0x61, 0x79, 0x2e, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x0b, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, - 0x72, 0x73, 0x12, 0x1b, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, - 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4b, 0x0a, - 0x0c, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x1c, 0x2e, - 0x67, 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, - 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1d, 0x2e, 0x67, 0x61, - 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x43, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x51, 0x0a, 0x0e, 0x55, 0x6e, - 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x1e, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x55, 0x6e, 0x63, 0x6f, 0x72, 0x64, 0x6f, 0x6e, 0x57, - 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, - 0x0b, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x12, 0x1b, 0x2e, 0x67, - 0x61, 0x74, 0x65, 0x77, 0x61, 0x79, 0x2e, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, - 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1c, 0x2e, 0x67, 0x61, 0x74, 0x65, - 0x77, 0x61, 0x79, 0x2e, 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x23, 0x5a, 0x21, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2f, 0x62, 0x65, 0x74, 0x61, 0x39, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x44, 0x72, 0x61, 0x69, 0x6e, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, + 0x6e, 0x73, 0x65, 0x42, 0x23, 0x5a, 0x21, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, + 0x6d, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x65, 0x74, + 0x61, 0x39, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto/taskqueue.pb.go b/proto/taskqueue.pb.go index ed775caf7..a5200ae01 100644 --- a/proto/taskqueue.pb.go +++ b/proto/taskqueue.pb.go @@ -442,7 +442,8 @@ type TaskQueueCompleteResponse struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"` + Ok bool `protobuf:"varint,1,opt,name=ok,proto3" json:"ok,omitempty"` + Message string `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"` } func (x *TaskQueueCompleteResponse) Reset() { @@ -484,6 +485,13 @@ func (x *TaskQueueCompleteResponse) GetOk() bool { return false } +func (x *TaskQueueCompleteResponse) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + type TaskQueueMonitorRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -979,104 +987,105 @@ var file_taskqueue_proto_rawDesc = []byte{ 0x65, 0x72, 0x48, 0x6f, 0x73, 0x74, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x11, 0x6b, 0x65, 0x65, 0x70, 0x5f, 0x77, 0x61, 0x72, 0x6d, 0x5f, 0x73, 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0f, 0x6b, 0x65, 0x65, 0x70, 0x57, 0x61, 0x72, 0x6d, 0x53, - 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x2b, 0x0a, 0x19, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, + 0x65, 0x63, 0x6f, 0x6e, 0x64, 0x73, 0x22, 0x45, 0x0a, 0x19, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, - 0x02, 0x6f, 0x6b, 0x22, 0x6e, 0x0a, 0x17, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, - 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, - 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, - 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, - 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, - 0x72, 0x49, 0x64, 0x22, 0x81, 0x01, 0x0a, 0x18, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, - 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x12, 0x1a, - 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, - 0x6d, 0x65, 0x64, 0x5f, 0x6f, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x74, - 0x69, 0x6d, 0x65, 0x64, 0x4f, 0x75, 0x74, 0x22, 0x4f, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x18, - 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x66, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, - 0x6f, 0x6e, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, - 0x22, 0x34, 0x0a, 0x19, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, - 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, - 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x22, 0x2c, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, - 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x02, 0x6f, 0x6b, 0x22, 0x53, 0x0a, 0x1e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, - 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x31, 0x0a, 0x1f, 0x54, 0x61, 0x73, - 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, - 0x6c, 0x69, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, - 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x32, 0x98, 0x06, 0x0a, - 0x10, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x75, - 0x74, 0x12, 0x1e, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, - 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x1f, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, - 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, 0x0a, 0x0c, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x50, 0x6f, 0x70, 0x12, 0x1e, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x6f, 0x70, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x6f, 0x70, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x54, 0x61, 0x73, 0x6b, 0x51, - 0x75, 0x65, 0x75, 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x12, 0x22, 0x2e, 0x74, 0x61, - 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, - 0x23, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, - 0x51, 0x75, 0x65, 0x75, 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, 0x12, 0x60, 0x0a, 0x11, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, - 0x65, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x74, 0x61, + 0x02, 0x6f, 0x6b, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x6e, 0x0a, + 0x17, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, 0x6f, + 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x74, 0x61, 0x73, 0x6b, + 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, + 0x64, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6f, + 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x22, 0x81, 0x01, + 0x0a, 0x18, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x12, 0x1c, 0x0a, 0x09, 0x63, 0x61, + 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x63, + 0x61, 0x6e, 0x63, 0x65, 0x6c, 0x6c, 0x65, 0x64, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x63, 0x6f, 0x6d, 0x70, + 0x6c, 0x65, 0x74, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x64, 0x5f, 0x6f, 0x75, + 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x74, 0x69, 0x6d, 0x65, 0x64, 0x4f, 0x75, + 0x74, 0x22, 0x4f, 0x0a, 0x1a, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, + 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x22, 0x66, 0x0a, 0x1b, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x51, + 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x64, 0x6f, 0x6e, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x64, 0x6f, 0x6e, 0x65, 0x12, 0x1b, 0x0a, + 0x09, 0x65, 0x78, 0x69, 0x74, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x08, 0x65, 0x78, 0x69, 0x74, 0x43, 0x6f, 0x64, 0x65, 0x22, 0x34, 0x0a, 0x19, 0x53, 0x74, + 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, + 0x22, 0x2c, 0x0a, 0x1a, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, + 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, + 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x22, 0x53, + 0x0a, 0x1e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, + 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x17, 0x0a, 0x07, 0x73, 0x74, 0x75, 0x62, 0x5f, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x09, 0x52, 0x06, 0x73, 0x74, 0x75, 0x62, 0x49, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x74, 0x69, 0x6d, + 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x74, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x22, 0x31, 0x0a, 0x1f, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x6f, 0x6b, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x02, 0x6f, 0x6b, 0x32, 0x98, 0x06, 0x0a, 0x10, 0x54, 0x61, 0x73, 0x6b, 0x51, + 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x51, 0x0a, 0x0c, 0x54, + 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x75, 0x74, 0x12, 0x1e, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x24, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, - 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x54, 0x61, 0x73, 0x6b, - 0x51, 0x75, 0x65, 0x75, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x21, 0x2e, 0x74, 0x61, + 0x65, 0x50, 0x75, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, - 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, + 0x65, 0x50, 0x75, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x51, + 0x0a, 0x0c, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x50, 0x6f, 0x70, 0x12, 0x1e, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, - 0x75, 0x65, 0x75, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, - 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x12, 0x25, 0x2e, 0x74, 0x61, - 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, - 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x53, - 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, - 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x63, - 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x12, 0x24, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, - 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x74, 0x61, 0x73, - 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, - 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x22, 0x00, 0x12, 0x72, 0x0a, 0x17, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, - 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x12, 0x29, + 0x75, 0x65, 0x75, 0x65, 0x50, 0x6f, 0x70, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1f, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, - 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, - 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x74, 0x61, 0x73, 0x6b, - 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, - 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x52, 0x65, 0x73, - 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x23, 0x5a, 0x21, 0x67, 0x69, 0x74, 0x68, 0x75, - 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x62, 0x65, 0x61, 0x6d, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, - 0x2f, 0x62, 0x65, 0x74, 0x61, 0x39, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x65, 0x75, 0x65, 0x50, 0x6f, 0x70, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x12, 0x5d, 0x0a, 0x10, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4d, 0x6f, + 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x12, 0x22, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, + 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4d, 0x6f, 0x6e, 0x69, 0x74, + 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x23, 0x2e, 0x74, 0x61, 0x73, 0x6b, + 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4d, + 0x6f, 0x6e, 0x69, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x30, 0x01, + 0x12, 0x60, 0x0a, 0x11, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x43, 0x6f, 0x6d, + 0x70, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x23, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, + 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, + 0x65, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e, 0x74, 0x61, 0x73, + 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x22, 0x00, 0x12, 0x5a, 0x0a, 0x0f, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4c, + 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x21, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, + 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4c, 0x65, 0x6e, 0x67, 0x74, + 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x22, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, + 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x4c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x68, + 0x0a, 0x13, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x12, 0x25, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, + 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, + 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x26, 0x2e, 0x74, + 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x61, + 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, + 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x63, 0x0a, 0x12, 0x53, 0x74, 0x6f, 0x70, + 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x12, 0x24, + 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x54, + 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x1a, 0x25, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x2e, 0x53, 0x74, 0x6f, 0x70, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x72, 0x0a, + 0x17, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, + 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x12, 0x29, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, + 0x75, 0x65, 0x75, 0x65, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, + 0x72, 0x76, 0x65, 0x4b, 0x65, 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x2a, 0x2e, 0x74, 0x61, 0x73, 0x6b, 0x71, 0x75, 0x65, 0x75, 0x65, 0x2e, + 0x54, 0x61, 0x73, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x65, 0x72, 0x76, 0x65, 0x4b, 0x65, + 0x65, 0x70, 0x41, 0x6c, 0x69, 0x76, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, + 0x00, 0x42, 0x23, 0x5a, 0x21, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x62, 0x65, 0x61, 0x6d, 0x2d, 0x63, 0x6c, 0x6f, 0x75, 0x64, 0x2f, 0x62, 0x65, 0x74, 0x61, 0x39, + 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/sdk/examples/demo/cr.py b/sdk/examples/demo/cr.py new file mode 100644 index 000000000..2e375f438 --- /dev/null +++ b/sdk/examples/demo/cr.py @@ -0,0 +1,36 @@ +from beta9 import Image, endpoint + + +def load(): + from transformers import AutoModelForCausalLM, AutoTokenizer + + model = AutoModelForCausalLM.from_pretrained('gpt2') + tokenizer = AutoTokenizer.from_pretrained('gpt2') + + return model, tokenizer + + +@endpoint( + # gpu='any', + # memory='12Gi', + cpu=1, + on_start=load, + workers=1, + image=(Image(python_packages=['transformers', 'torch'])), + checkpoint_enabled=True, +) +def inference(context, prompt): + model, tokenizer = context.on_start_value + + # Generate + inputs = tokenizer(prompt, return_tensors='pt') + generate_ids = model.generate(inputs.input_ids, max_length=30) + result = tokenizer.batch_decode( + generate_ids, + skip_special_tokens=True, + clean_up_tokenization_spaces=False, + )[0] + + print(result) + + return {'prediction': result} diff --git a/sdk/pyproject.toml b/sdk/pyproject.toml index 68c80fade..dc625348d 100644 --- a/sdk/pyproject.toml +++ b/sdk/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "beta9" -version = "0.1.125" +version = "0.1.131" description = "" authors = ["beam.cloud "] packages = [ diff --git a/sdk/src/beta9/abstractions/base/runner.py b/sdk/src/beta9/abstractions/base/runner.py index 21c827352..59382e923 100644 --- a/sdk/src/beta9/abstractions/base/runner.py +++ b/sdk/src/beta9/abstractions/base/runner.py @@ -79,6 +79,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: Union[GpuTypeAlias, List[GpuTypeAlias]] = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), workers: int = 1, concurrent_requests: int = 1, @@ -94,6 +95,7 @@ def __init__( name: Optional[str] = None, autoscaler: Autoscaler = QueueDepthAutoscaler(), task_policy: TaskPolicy = TaskPolicy(), + checkpoint_enabled: bool = False, ) -> None: super().__init__() @@ -116,6 +118,7 @@ def __init__( self.cpu = cpu self.memory = self._parse_memory(memory) if isinstance(memory, str) else memory self.gpu = gpu + self.gpu_count = gpu_count self.volumes = volumes or [] self.secrets = [SecretVar(name=s) for s in (secrets or [])] self.workers = workers @@ -128,8 +131,12 @@ def __init__( timeout=task_policy.timeout or timeout, ttl=task_policy.ttl, ) + self.checkpoint_enabled = checkpoint_enabled self.extra: dict = {} + if (self.gpu != "" or len(self.gpu) > 0) and self.gpu_count == 0: + self.gpu_count = 1 + if on_start is not None: self._map_callable_to_attr(attr="on_start", func=on_start) @@ -402,6 +409,7 @@ def prepare_runtime( cpu=self.cpu, memory=self.memory, gpu=self.gpu, + gpu_count=self.gpu_count, handler=self.handler, on_start=self.on_start, callback_url=self.callback_url, @@ -423,6 +431,7 @@ def prepare_runtime( ttl=self.task_policy.ttl, ), concurrent_requests=self.concurrent_requests, + checkpoint_enabled=self.checkpoint_enabled, extra=json.dumps(self.extra), ) if _is_stub_created_for_workspace(): diff --git a/sdk/src/beta9/abstractions/container.py b/sdk/src/beta9/abstractions/container.py index e9f981c5c..5e5b9e25e 100644 --- a/sdk/src/beta9/abstractions/container.py +++ b/sdk/src/beta9/abstractions/container.py @@ -32,6 +32,9 @@ class Container(RunnerAbstraction): applicable or no GPU required, leave it empty. You can specify multiple GPUs by providing a list of GpuTypeAlias. If you specify several GPUs, the scheduler prioritizes their selection based on their order in the list. + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). volumes (Optional[List[Volume]]): @@ -61,6 +64,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: Union[GpuTypeAlias, List[GpuTypeAlias]] = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), volumes: Optional[List[Volume]] = None, secrets: Optional[List[str]] = None, @@ -70,6 +74,7 @@ def __init__( cpu=cpu, memory=memory, gpu=gpu, + gpu_count=gpu_count, image=image, volumes=volumes, secrets=secrets, diff --git a/sdk/src/beta9/abstractions/endpoint.py b/sdk/src/beta9/abstractions/endpoint.py index 9df029bd4..b25651b59 100644 --- a/sdk/src/beta9/abstractions/endpoint.py +++ b/sdk/src/beta9/abstractions/endpoint.py @@ -1,6 +1,7 @@ import os import threading import traceback +import types from typing import Any, Callable, List, Optional, Union from uvicorn.protocols.utils import ClientDisconnected @@ -46,6 +47,9 @@ class Endpoint(RunnerAbstraction): applicable or no GPU required, leave it empty. You can specify multiple GPUs by providing a list of GpuTypeAlias. If you specify several GPUs, the scheduler prioritizes their selection based on their order in the list. + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). volumes (Optional[List[Volume]]): @@ -85,6 +89,11 @@ class Endpoint(RunnerAbstraction): task_policy (TaskPolicy): The task policy for the function. This helps manage the lifecycle of an individual task. Setting values here will override timeout and retries. + checkpoint_enabled (bool): + (experimental) Whether to enable checkpointing for the endpoint. Default is False. + If enabled, the app will be checkpointed after the on_start function has completed. + On next invocation, each container will restore from a checkpoint and resume execution instead of + booting up from cold. Example: ```python from beta9 import endpoint, Image @@ -110,6 +119,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: Union[GpuTypeAlias, List[GpuTypeAlias]] = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), timeout: int = 180, workers: int = 1, @@ -123,11 +133,13 @@ def __init__( autoscaler: Autoscaler = QueueDepthAutoscaler(), callback_url: Optional[str] = None, task_policy: TaskPolicy = TaskPolicy(), + checkpoint_enabled: bool = False, ): super().__init__( cpu=cpu, memory=memory, gpu=gpu, + gpu_count=gpu_count, image=image, workers=workers, timeout=timeout, @@ -143,6 +155,7 @@ def __init__( callback_url=callback_url, task_policy=task_policy, concurrent_requests=self.concurrent_requests, + checkpoint_enabled=checkpoint_enabled, ) self._endpoint_stub: Optional[EndpointServiceStub] = None @@ -170,6 +183,9 @@ class ASGI(Endpoint): gpu (Union[GpuType, str]): The type or name of the GPU device to be used for GPU-accelerated tasks. If not applicable or no GPU required, leave it empty. Default is [GpuType.NoGPU](#gputype). + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). volumes (Optional[List[Volume]]): @@ -216,6 +232,11 @@ class ASGI(Endpoint): task_policy (TaskPolicy): The task policy for the function. This helps manage the lifecycle of an individual task. Setting values here will override timeout and retries. + checkpoint_enabled (bool): + (experimental) Whether to enable checkpointing for the endpoint. Default is False. + If enabled, the app will be checkpointed after the on_start function has completed. + On next invocation, each container will restore from a checkpoint and resume execution instead of + booting up from cold. Example: ```python from beta9 import asgi, Image @@ -247,6 +268,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: GpuTypeAlias = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), timeout: int = 180, workers: int = 1, @@ -260,12 +282,14 @@ def __init__( authorized: bool = True, autoscaler: Autoscaler = QueueDepthAutoscaler(), callback_url: Optional[str] = None, + checkpoint_enabled: bool = False, ): self.concurrent_requests = concurrent_requests super().__init__( cpu=cpu, memory=memory, gpu=gpu, + gpu_count=gpu_count, image=image, timeout=timeout, workers=workers, @@ -278,6 +302,7 @@ def __init__( authorized=authorized, autoscaler=autoscaler, callback_url=callback_url, + checkpoint_enabled=checkpoint_enabled, ) self.is_asgi = True @@ -301,6 +326,9 @@ class RealtimeASGI(ASGI): gpu (Union[GpuType, str]): The type or name of the GPU device to be used for GPU-accelerated tasks. If not applicable or no GPU required, leave it empty. Default is [GpuType.NoGPU](#gputype). + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). volumes (Optional[List[Volume]]): @@ -342,6 +370,11 @@ class RealtimeASGI(ASGI): various autoscaling strategies (Defaults to QueueDepthAutoscaler()) callback_url (Optional[str]): An optional URL to send a callback to when a task is completed, timed out, or cancelled. + checkpoint_enabled (bool): + (experimental) Whether to enable checkpointing for the endpoint. Default is False. + If enabled, the app will be checkpointed after the on_start function has completed. + On next invocation, each container will restore from a checkpoint and resume execution instead of + booting up from cold. Example: ```python from beta9 import realtime @@ -364,6 +397,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: GpuTypeAlias = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), timeout: int = 180, workers: int = 1, @@ -377,11 +411,13 @@ def __init__( authorized: bool = True, autoscaler: Autoscaler = QueueDepthAutoscaler(), callback_url: Optional[str] = None, + checkpoint_enabled: bool = False, ): super().__init__( cpu=cpu, memory=memory, gpu=gpu, + gpu_count=gpu_count, image=image, timeout=timeout, workers=workers, @@ -395,6 +431,7 @@ def __init__( autoscaler=autoscaler, callback_url=callback_url, concurrent_requests=concurrent_requests, + checkpoint_enabled=checkpoint_enabled, ) self.is_websocket = True @@ -439,18 +476,25 @@ async def _heartbeat(): internal_asgi_app.input_queue.put(data) + async def _handle_output(output): + if isinstance(output, str): + await websocket.send_text(output) + elif isinstance(output, dict) or isinstance(output, list): + await websocket.send_json(output) + else: + await websocket.send(output) + while not internal_asgi_app.input_queue.empty(): output = internal_asgi_app.handler( context=internal_asgi_app.context, event=internal_asgi_app.input_queue.get(), ) - if isinstance(output, str): - await websocket.send_text(output) - elif isinstance(output, dict) or isinstance(output, list): - await websocket.send_json(output) + if isinstance(output, types.GeneratorType): + for o in output: + await _handle_output(o) else: - await websocket.send_bytes(output) + await _handle_output(output) await asyncio.sleep(REALTIME_ASGI_SLEEP_INTERVAL_SECONDS) except ( diff --git a/sdk/src/beta9/abstractions/experimental/bot/bot.py b/sdk/src/beta9/abstractions/experimental/bot/bot.py index 4abd46ae8..cce9e9acf 100644 --- a/sdk/src/beta9/abstractions/experimental/bot/bot.py +++ b/sdk/src/beta9/abstractions/experimental/bot/bot.py @@ -237,6 +237,8 @@ class Bot(RunnerAbstraction, DeployableMixin): authorized (bool): If false, allows the bot to be invoked without an auth token. Default is True. + welcome_message (Optional[str]): + A welcome message to display to a user when a new session with the bot is started. Default is None. """ deployment_stub_type = BOT_DEPLOYMENT_STUB_TYPE @@ -261,6 +263,7 @@ def __init__( description: Optional[str] = None, volumes: Optional[List[Volume]] = None, authorized: bool = True, + welcome_message: Optional[str] = None, ) -> None: super().__init__(volumes=volumes) @@ -284,6 +287,7 @@ def __init__( self.extra["description"] = description self.extra["api_key"] = api_key self.extra["authorized"] = authorized + self.extra["welcome_message"] = welcome_message for location in self.locations: location_config = location.to_dict() diff --git a/sdk/src/beta9/abstractions/function.py b/sdk/src/beta9/abstractions/function.py index 4e3cb85a4..65ec006fb 100644 --- a/sdk/src/beta9/abstractions/function.py +++ b/sdk/src/beta9/abstractions/function.py @@ -44,6 +44,9 @@ class Function(RunnerAbstraction): applicable or no GPU required, leave it empty. You can specify multiple GPUs by providing a list of GpuTypeAlias. If you specify several GPUs, the scheduler prioritizes their selection based on their order in the list. + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). timeout (Optional[int]): @@ -63,6 +66,8 @@ class Function(RunnerAbstraction): task_policy (TaskPolicy): The task policy for the function. This helps manage the lifecycle of an individual task. Setting values here will override timeout and retries. + retry_for (Optional[List[BaseException]]): + A list of exceptions that will trigger a retry. Example: ```python from beta9 import function, Image @@ -87,6 +92,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: Union[GpuTypeAlias, List[GpuTypeAlias]] = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), timeout: int = 3600, retries: int = 3, @@ -95,11 +101,13 @@ def __init__( secrets: Optional[List[str]] = None, name: Optional[str] = None, task_policy: TaskPolicy = TaskPolicy(), + retry_for: Optional[List[Exception]] = None, ) -> None: super().__init__( cpu=cpu, memory=memory, gpu=gpu, + gpu_count=gpu_count, image=image, timeout=timeout, retries=retries, @@ -112,6 +120,7 @@ def __init__( self._function_stub: Optional[FunctionServiceStub] = None self.syncer: FileSyncer = FileSyncer(self.gateway_stub) + self.retry_for = retry_for def __call__(self, func): return _CallableWrapper(func, self) @@ -283,6 +292,9 @@ class Schedule(Function): gpu (Union[GpuType, str]): The type or name of the GPU device to be used for GPU-accelerated tasks. If not applicable or no GPU required, leave it empty. Default is [GpuType.NoGPU](#gputype). + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). timeout (Optional[int]): @@ -326,6 +338,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: GpuTypeAlias = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), timeout: int = 3600, retries: int = 3, diff --git a/sdk/src/beta9/abstractions/integrations/vllm.py b/sdk/src/beta9/abstractions/integrations/vllm.py index 7bc8625a9..a66703c23 100644 --- a/sdk/src/beta9/abstractions/integrations/vllm.py +++ b/sdk/src/beta9/abstractions/integrations/vllm.py @@ -32,12 +32,18 @@ class VLLMArgs: Each of these arguments corresponds to a command line argument for the vllm server. """ + # Args for init_app_state response_role: Optional[str] = "assistant" lora_modules: Optional[List[str]] = None prompt_adapters: Optional[List[str]] = None chat_template: Optional[str] = None chat_template_url: Optional[str] = None + chat_template_text_format: str = "string" + allowed_local_media_path: str = "" + hf_overrides: Optional[Union[Dict[str, Any], Callable[[Any], Any]]] = None + enable_lora_bias: bool = False return_tokens_as_token_ids: bool = False + enable_prompt_tokens_details: bool = False enable_auto_tool_choice: bool = False tool_call_parser: Optional[str] = None tool_parser_plugin: Optional[str] = None @@ -47,6 +53,8 @@ class VLLMArgs: model: str = "facebook/opt-125m" served_model_name: Optional[Union[str, List[str]]] = None tokenizer: Optional[str] = None + + # Args for AsyncEngineArgs skip_tokenizer_init: bool = False tokenizer_mode: str = "auto" task: str = "auto" @@ -74,7 +82,6 @@ class VLLMArgs: max_num_batched_tokens: Optional[int] = None max_num_seqs: int = 256 max_logprobs: int = 20 - disable_log_stats: bool = False revision: Optional[str] = None code_revision: Optional[str] = None rope_scaling: Optional[dict] = None @@ -130,6 +137,7 @@ class VLLMArgs: collect_detailed_traces: Optional[str] = None disable_async_output_proc: bool = False override_neuron_config: Optional[Dict[str, Any]] = None + override_pooler_config: Optional[Any] = None mm_processor_kwargs: Optional[Dict[str, Any]] = None scheduling_policy: Literal["fcfs", "priority"] = "fcfs" disable_log_requests: bool = False @@ -209,7 +217,9 @@ def __init__( # Add default vllm cache volume to preserve it if custom volumes are specified for chat templates volumes.append(Volume(name="vllm_cache", mount_path=DEFAULT_VLLM_CACHE_DIR)) - image = image.add_python_packages(["fastapi", "vllm", "huggingface_hub"]) + image = image.add_python_packages( + ["fastapi", "numpy", "vllm==0.6.4.post1", "huggingface_hub==0.26.3"] + ) super().__init__( cpu=cpu, diff --git a/sdk/src/beta9/abstractions/taskqueue.py b/sdk/src/beta9/abstractions/taskqueue.py index 55fc67c66..159b9a12b 100644 --- a/sdk/src/beta9/abstractions/taskqueue.py +++ b/sdk/src/beta9/abstractions/taskqueue.py @@ -44,6 +44,9 @@ class TaskQueue(RunnerAbstraction): applicable or no GPU required, leave it empty. You can specify multiple GPUs by providing a list of GpuTypeAlias. If you specify several GPUs, the scheduler prioritizes their selection based on their order in the list. + gpu_count (int): + The number of GPUs allocated to the container. Default is 0. If a GPU is + specified but this value is set to 0, it will be automatically updated to 1. image (Union[Image, dict]): The container image used for the task execution. Default is [Image](#image). timeout (Optional[int]): @@ -86,6 +89,13 @@ class TaskQueue(RunnerAbstraction): task_policy (TaskPolicy): The task policy for the function. This helps manage the lifecycle of an individual task. Setting values here will override timeout and retries. + retry_for (Optional[List[BaseException]]): + A list of exceptions that will trigger a retry if raised by your handler. + checkpoint_enabled (bool): + (experimental) Whether to enable checkpointing for the task queue. Default is False. + If enabled, the app will be checkpointed after the on_start function has completed. + On next invocation, each container will restore from a checkpoint and resume execution instead of + booting up from cold. Example: ```python from beta9 import task_queue, Image @@ -105,6 +115,7 @@ def __init__( cpu: Union[int, float, str] = 1.0, memory: Union[int, str] = 128, gpu: Union[GpuTypeAlias, List[GpuTypeAlias]] = GpuType.NoGPU, + gpu_count: int = 0, image: Image = Image(), timeout: int = 3600, retries: int = 3, @@ -119,11 +130,14 @@ def __init__( authorized: bool = True, autoscaler: Autoscaler = QueueDepthAutoscaler(), task_policy: TaskPolicy = TaskPolicy(), + checkpoint_enabled: bool = False, + retry_for: Optional[List[BaseException]] = None, ) -> None: super().__init__( cpu=cpu, memory=memory, gpu=gpu, + gpu_count=gpu_count, image=image, workers=workers, timeout=timeout, @@ -138,8 +152,10 @@ def __init__( authorized=authorized, autoscaler=autoscaler, task_policy=task_policy, + checkpoint_enabled=checkpoint_enabled, ) self._taskqueue_stub: Optional[TaskQueueServiceStub] = None + self.retry_for = retry_for @property def taskqueue_stub(self) -> TaskQueueServiceStub: diff --git a/sdk/src/beta9/clients/gateway/__init__.py b/sdk/src/beta9/clients/gateway/__init__.py index 0e303edf4..1a0c672c6 100644 --- a/sdk/src/beta9/clients/gateway/__init__.py +++ b/sdk/src/beta9/clients/gateway/__init__.py @@ -282,6 +282,8 @@ class GetOrCreateStubRequest(betterproto.Message): task_policy: "TaskPolicy" = betterproto.message_field(23) concurrent_requests: int = betterproto.uint32_field(24) extra: str = betterproto.string_field(25) + checkpoint_enabled: bool = betterproto.bool_field(26) + gpu_count: int = betterproto.uint32_field(27) @dataclass(eq=False, repr=False) diff --git a/sdk/src/beta9/clients/taskqueue/__init__.py b/sdk/src/beta9/clients/taskqueue/__init__.py index b76279acb..c78e591c3 100644 --- a/sdk/src/beta9/clients/taskqueue/__init__.py +++ b/sdk/src/beta9/clients/taskqueue/__init__.py @@ -73,6 +73,7 @@ class TaskQueueCompleteRequest(betterproto.Message): @dataclass(eq=False, repr=False) class TaskQueueCompleteResponse(betterproto.Message): ok: bool = betterproto.bool_field(1) + message: str = betterproto.string_field(2) @dataclass(eq=False, repr=False) diff --git a/sdk/src/beta9/middleware.py b/sdk/src/beta9/middleware.py index 8fb425cd7..dbd72871a 100644 --- a/sdk/src/beta9/middleware.py +++ b/sdk/src/beta9/middleware.py @@ -33,13 +33,21 @@ async def run_task(request, func, func_args): os.environ["TASK_ID"] = task_id with StdoutJsonInterceptor(task_id=task_id): print(f"Received task <{task_id}>") - start_response = request.app.state.gateway_stub.start_task( - StartTaskRequest(task_id=task_id, container_id=cfg.container_id) - ) - if not start_response.ok: - raise HTTPException( - status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Failed to start task" - ) + + for attempt in range(3): + try: + start_response = request.app.state.gateway_stub.start_task( + StartTaskRequest(task_id=task_id, container_id=cfg.container_id) + ) + if start_response.ok: + break + else: + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Failed to start task" + ) + except BaseException: + if attempt == 2: + raise task_lifecycle_data = TaskLifecycleData( status=TaskStatus.Complete, result=None, override_callback_url=None diff --git a/sdk/src/beta9/runner/bot/transition.py b/sdk/src/beta9/runner/bot/transition.py index af77c8d9e..1ebaae958 100644 --- a/sdk/src/beta9/runner/bot/transition.py +++ b/sdk/src/beta9/runner/bot/transition.py @@ -67,12 +67,15 @@ def _format_inputs(self, markers: Dict[str, Any]) -> Dict[str, Any]: if field_name in marker_class.model_fields: field_type = marker_class.model_fields[field_name].annotation else: - field_type = str # default to string if field not defined + field_type = str # default to string if field is not defined # Try to convert field_value to field_type try: - converted_value = field_type(field_value) - except (ValueError, TypeError): + if get_origin(field_type) is dict and isinstance(field_value, str): + converted_value = json.loads(field_value) + else: + converted_value = field_type(field_value) + except (ValueError, TypeError, json.JSONDecodeError): converted_value = field_value fields_dict[field_name] = converted_value diff --git a/sdk/src/beta9/runner/common.py b/sdk/src/beta9/runner/common.py index 5d9021129..483fe10ca 100644 --- a/sdk/src/beta9/runner/common.py +++ b/sdk/src/beta9/runner/common.py @@ -10,6 +10,8 @@ from contextlib import contextmanager from dataclasses import dataclass from functools import wraps +from multiprocessing import Value +from pathlib import Path from typing import Any, Callable, Dict, Optional, Union import requests @@ -45,6 +47,7 @@ class Config: callback_url: str task_id: str bind_port: int + checkpoint_enabled: bool volume_cache_map: Dict @classmethod @@ -62,6 +65,7 @@ def load_from_env(cls) -> "Config": task_id = os.getenv("TASK_ID") bind_port = int(os.getenv("BIND_PORT")) timeout = int(os.getenv("TIMEOUT", 180)) + checkpoint_enabled = os.getenv("CHECKPOINT_ENABLED", "false").lower() == "true" volume_cache_map = json.loads(os.getenv("VOLUME_CACHE_MAP", "{}")) if workers <= 0: @@ -84,6 +88,7 @@ def load_from_env(cls) -> "Config": task_id=task_id, bind_port=bind_port, timeout=timeout, + checkpoint_enabled=checkpoint_enabled, volume_cache_map=volume_cache_map, ) @@ -93,6 +98,30 @@ def load_from_env(cls) -> "Config": config: Config = Config.load_from_env() +class ParentAbstractionProxy: + """ + Class to allow handlers to access parent class variables through attribute or dictionary access + """ + + def __init__(self, parent): + self._parent = parent + + def __getitem__(self, key): + return getattr(self._parent, key) + + def __setitem__(self, key, value): + setattr(self._parent, key, value) + + def __getattr__(self, key): + return getattr(self._parent, key) + + def __setattr__(self, key, value): + if key == "_parent": + super().__setattr__(key, value) + else: + setattr(self._parent, key, value) + + @dataclass class FunctionContext: """ @@ -133,6 +162,11 @@ def new( ) +workers_ready = None +if is_remote(): + workers_ready = Value("i", 0) + + class FunctionHandler: """ Helper class for loading user entry point functions @@ -184,6 +218,12 @@ def __call__(self, context: FunctionContext, *args: Any, **kwargs: Any) -> Any: os.environ["TASK_ID"] = context.task_id or "" return self.handler(*args, **kwargs) + @property + def parent_abstraction(self) -> ParentAbstractionProxy: + if not hasattr(self, "_parent_abstraction"): + self._parent_abstraction = ParentAbstractionProxy(self.handler.parent) + return self._parent_abstraction + def execute_lifecycle_method(name: str) -> Union[Any, None]: """Executes a container lifecycle method defined by the user and return it's value""" @@ -344,3 +384,35 @@ def __exit__(self, *_, **__): self.shutdown(cancel_futures=True) except Exception: pass + + +CHECKPOINT_SIGNAL_FILE = "/cedana/READY_FOR_CHECKPOINT" +CHECKPOINT_COMPLETE_FILE = "/cedana/CHECKPOINT_COMPLETE" +CHECKPOINT_CONTAINER_ID_FILE = "/cedana/CONTAINER_ID" +CHECKPOINT_CONTAINER_HOSTNAME_FILE = "/cedana/CONTAINER_HOSTNAME" + + +def wait_for_checkpoint(): + def _reload_config(): + # Once we have set the checkpoint signal file, wait for checkpoint to be complete before reloading the config + while not Path(CHECKPOINT_COMPLETE_FILE).exists(): + time.sleep(1) + + # Reload config that may have changed during restore + config.container_id = Path(CHECKPOINT_CONTAINER_ID_FILE).read_text() + config.container_hostname = Path(CHECKPOINT_CONTAINER_HOSTNAME_FILE).read_text() + + with workers_ready.get_lock(): + workers_ready.value += 1 + + if workers_ready.value == config.workers: + Path(CHECKPOINT_SIGNAL_FILE).touch(exist_ok=True) + return _reload_config() + + while True: + with workers_ready.get_lock(): + if workers_ready.value == config.workers: + break + time.sleep(1) + + return _reload_config() diff --git a/sdk/src/beta9/runner/endpoint.py b/sdk/src/beta9/runner/endpoint.py index a86bac1d2..6a70a312a 100644 --- a/sdk/src/beta9/runner/endpoint.py +++ b/sdk/src/beta9/runner/endpoint.py @@ -28,7 +28,12 @@ TaskLifecycleMiddleware, WebsocketTaskLifecycleMiddleware, ) -from ..runner.common import FunctionContext, FunctionHandler, execute_lifecycle_method +from ..runner.common import ( + FunctionContext, + FunctionHandler, + execute_lifecycle_method, + wait_for_checkpoint, +) from ..runner.common import config as cfg from ..type import LifeCycleMethod, TaskStatus from .common import is_asgi3 @@ -88,6 +93,11 @@ def post_fork_initialize(_, worker: UvicornWorker): # Override the default starlette app worker.app.callable = asgi_app + + # If checkpointing is enabled, wait for all workers to be ready before creating a checkpoint + if cfg.checkpoint_enabled: + wait_for_checkpoint() + except EOFError: return except BaseException: diff --git a/sdk/src/beta9/runner/taskqueue.py b/sdk/src/beta9/runner/taskqueue.py index 0f39d03ac..f8a0c9010 100644 --- a/sdk/src/beta9/runner/taskqueue.py +++ b/sdk/src/beta9/runner/taskqueue.py @@ -33,7 +33,9 @@ config, execute_lifecycle_method, send_callback, + wait_for_checkpoint, ) +from ..runner.common import config as cfg from ..type import LifeCycleMethod, TaskExitCode, TaskStatus TASK_PROCESS_WATCHDOG_INTERVAL = 0.01 @@ -272,6 +274,11 @@ def process_tasks(self, channel: Channel) -> None: on_start_value = execute_lifecycle_method(name=LifeCycleMethod.OnStart) print(f"Worker[{self.worker_index}] ready") + + # If checkpointing is enabled, wait for all workers to be ready before creating a checkpoint + if cfg.checkpoint_enabled: + wait_for_checkpoint() + with ThreadPoolExecutorOverride() as thread_pool: while True: task = self._get_next_task(taskqueue_stub, config.stub_id, config.container_id) @@ -304,14 +311,21 @@ def process_tasks(self, channel: Channel) -> None: result = None duration = None + caught_exception = "" + try: args = task.args or [] kwargs = task.kwargs or {} result = handler(context, *args, **kwargs) - except BaseException: + except BaseException as e: print(traceback.format_exc()) - task_status = TaskStatus.Error + + if type(e) in handler.parent_abstraction.retry_for: + caught_exception = e.__class__.__name__ + task_status = TaskStatus.Retry + else: + task_status = TaskStatus.Error finally: duration = time.time() - start_time @@ -333,15 +347,22 @@ def process_tasks(self, channel: Channel) -> None: if not complete_task_response.ok: raise RunnerException("Unable to end task") - print(f"Task completed <{task.id}>, took {duration}s") + if task_status == TaskStatus.Retry: + print( + complete_task_response.message + or f"Retrying task <{task.id}> after {caught_exception} exception" + ) + continue + print(f"Task completed <{task.id}>, took {duration}s") send_callback( gateway_stub=gateway_stub, context=context, payload=result or {}, task_status=task_status, override_callback_url=kwargs.get("callback_url"), - ) # Send callback to callback_url, if defined + ) + except BaseException: print(traceback.format_exc()) finally: