From adaa867f2eb2fcb0277ecd1221230f960d6b8447 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 4 Oct 2024 02:36:36 -0400 Subject: [PATCH 01/44] checkpoint - kicking off cloud build from cli sorta working --- src/deploy.ts | 141 +++++++++++++++++++++++++++++++++++-- src/observableApiClient.ts | 57 +++++++++++++++ 2 files changed, 192 insertions(+), 6 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index dbf19540d..ddd704b15 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -1,7 +1,10 @@ +import {exec} from "node:child_process"; import {createHash} from "node:crypto"; import type {Stats} from "node:fs"; +import {existsSync} from "node:fs"; import {readFile, stat} from "node:fs/promises"; import {join} from "node:path/posix"; +import {promisify} from "node:util"; import slugify from "@sindresorhus/slugify"; import wrapAnsi from "wrap-ansi"; import type {BuildEffects, BuildManifest, BuildOptions} from "./build.js"; @@ -20,6 +23,7 @@ import type { DeployManifestFile, GetCurrentUserResponse, GetDeployResponse, + GetProjectEnvironmentResponse, GetProjectResponse, WorkspaceResponse } from "./observableApiClient.js"; @@ -33,6 +37,10 @@ const DEPLOY_POLL_MAX_MS = 1000 * 60 * 5; const DEPLOY_POLL_INTERVAL_MS = 1000 * 5; const BUILD_AGE_WARNING_MS = 1000 * 60 * 5; +export function formatGitUrl(url: string) { + return new URL(url).pathname.slice(1).replace(/\.git$/, ""); +} + export interface DeployOptions { config: Config; deployConfigPath: string | undefined; @@ -82,9 +90,14 @@ const defaultEffects: DeployEffects = { type DeployTargetInfo = | {create: true; workspace: {id: string; login: string}; projectSlug: string; title: string; accessLevel: string} - | {create: false; workspace: {id: string; login: string}; project: GetProjectResponse}; - -/** Deploy a project to ObservableHQ */ + | { + create: false; + workspace: {id: string; login: string}; + project: GetProjectResponse; + environment: GetProjectEnvironmentResponse; + }; + +/** Deploy a project to Observable */ export async function deploy(deployOptions: DeployOptions, effects = defaultEffects): Promise { Telemetry.record({event: "deploy", step: "start", force: deployOptions.force}); effects.clack.intro(`${inverse(" observable deploy ")} ${faint(`v${process.env.npm_package_version}`)}`); @@ -190,9 +203,104 @@ class Deployer { return deployInfo; } + private async maybeCloudBuild(deployTarget: DeployTargetInfo) { + if (deployTarget.create) return false; + const confirmCloudBuild = await this.effects.clack.confirm({ + message: "Do you want to build in the cloud?", + active: "Yes", + inactive: "No" + }); + if (confirmCloudBuild) { + // kick off new cloud deploy w/ link to + const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; + await this.apiClient.postProjectBuild(deployTarget.project.id); + const spinner = this.effects.clack.spinner(); + spinner.start("Requesting deploy…"); + const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; + pollLoop: while (true) { + if (Date.now() > pollExpiration) { + spinner.stop("Requesting deploy timed out"); + throw new CliError("Requesting deploy failed"); + } + const {latestCreatedDeployId} = await this.apiClient.getProject({ + workspaceLogin: deployTarget.workspace.login, + projectSlug: deployTarget.project.slug + }); + if (latestCreatedDeployId !== deployTarget.project.latestCreatedDeployId) { + // TODO use observable_origin env var? + spinner.stop( + `Deploy started. Watch logs: https://observable.test:5000/projects/@${deployTarget.workspace.login}/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` + ); + break pollLoop; + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); + } + return true; + } + return false; + } + + private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { + if (!this.effects.isTty || deployTarget.create) return false; + if (deployTarget.environment.build_environment_id && deployTarget.environment.source) { + // can do cloud build + return true; + } else { + // TODO Where should it look for .git? + // const isGit = existsSync(this.deployOptions.config.root + "/.git"); + const isGit = existsSync(".git"); + if (isGit) { + const remotes = (await promisify(exec)("git remote -v", {cwd: this.deployOptions.config.root})).stdout + .split("\n") + .filter((d) => d) + .map((d) => d.split(/\s/g)); + const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); + if (gitHub) { + const repoName = formatGitUrl(gitHub[1]); + const confirmLinkGitHub = await this.effects.clack.confirm({ + message: `Do you want to link to GitHub repo ${repoName}?`, + active: "Yes", + inactive: "No" + }); + if (confirmLinkGitHub) { + const {repositories} = await this.apiClient.getGitHubRepositories(); + const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); + if (authedRepo) { + // authed repo found + await this.apiClient.postProjectEnvironment(deployTarget.project.id, { + source: { + provider: authedRepo.provider, + provider_id: authedRepo.provider_id, + url: authedRepo.url, + branch: null // TODO detect branch + } + }); + return true; + } else { + // repo not auth’ed; kick off web auth flow + this.effects.clack.log.info( + "Authorize Observable to access this repo: https://github.com/apps/observable-data-apps-dev/installations/select_target" + ); + } + } + } else { + // no github remote + } + } else { + // not a repo + } + } + return false; + } + private async startNewDeploy(): Promise { const deployConfig = await this.getUpdatedDeployConfig(); const deployTarget = await this.getDeployTarget(deployConfig); + const linkedGitHub = await this.maybeLinkGitHub(deployTarget); + if (linkedGitHub) { + const cloudBuild = await this.maybeCloudBuild(deployTarget); + if (cloudBuild) return true; + } const buildFilePaths = await this.getBuildFilePaths(); const deployId = await this.createNewDeploy(deployTarget); @@ -282,7 +390,8 @@ class Deployer { workspaceLogin: deployConfig.workspaceLogin, projectSlug: deployConfig.projectSlug }); - deployTarget = {create: false, workspace: project.owner, project}; + const environment = await this.apiClient.getProjectEnvironment({id: project.id}); + deployTarget = {create: false, workspace: project.owner, project, environment}; } catch (error) { if (!isHttpError(error) || error.statusCode !== 404) { throw error; @@ -360,7 +469,17 @@ class Deployer { workspaceId: deployTarget.workspace.id, accessLevel: deployTarget.accessLevel }); - deployTarget = {create: false, workspace: deployTarget.workspace, project}; + // TODO(toph): initial env config + deployTarget = { + create: false, + workspace: deployTarget.workspace, + project, + environment: { + automatic_builds_enabled: null, + build_environment_id: null, + source: null + } + }; } catch (error) { if (isApiError(error) && error.details.errors.some((e) => e.code === "TOO_MANY_PROJECTS")) { this.effects.clack.log.error( @@ -756,7 +875,17 @@ export async function promptDeployTarget( if (effects.clack.isCancel(chosenProject)) { throw new CliError("User canceled deploy.", {print: false, exitCode: 0}); } else if (chosenProject !== null) { - return {create: false, workspace, project: existingProjects.find((p) => p.slug === chosenProject)!}; + // TODO(toph): initial env config + return { + create: false, + workspace, + project: existingProjects.find((p) => p.slug === chosenProject)!, + environment: { + automatic_builds_enabled: null, + build_environment_id: null, + source: null + } + }; } } else { const confirmChoice = await effects.clack.confirm({ diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index fe8b2b38a..990c7d5f8 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -126,6 +126,31 @@ export class ObservableApiClient { return await this._fetch(url, {method: "GET"}); } + async getProjectEnvironment({id}: {id: string}): Promise { + const url = new URL(`/cli/project/${id}/environment`, this._apiOrigin); + return await this._fetch(url, {method: "GET"}); + } + + async getGitHubRepositories(): Promise { + const url = new URL("/cli/github/repositories", this._apiOrigin); + return await this._fetch(url, {method: "GET"}); + } + + async postProjectEnvironment(id, body): Promise { + const url = new URL(`/cli/project/${id}/environment`, this._apiOrigin); + return await this._fetch(url, { + method: "POST", + headers: {"Content-Type": "application/json"}, + body: JSON.stringify(body) + }); + } + + async postProjectBuild(id): Promise<{id: string}> { + return await this._fetch<{id: string}>(new URL(`/cli/project/${id}/build`, this._apiOrigin), { + method: "POST" + }); + } + async postProject({ title, slug, @@ -264,10 +289,42 @@ export interface GetProjectResponse { title: string; owner: {id: string; login: string}; creator: {id: string; login: string}; + latestCreatedDeployId: string | null; // Available fields that we don't use // servingRoot: string | null; } +export interface GetProjectEnvironmentResponse { + automatic_builds_enabled: boolean | null; + build_environment_id: string | null; + source: null | { + provider: string; + provider_id: string; + url: string; + branch: string | null; + }; +} + +export interface GetGitHubRepositoriesResponse { + installations: { + id: number; + login: string | null; + name: string | null; + }[]; + repositories: { + provider: "github"; + provider_id: string; + url: string; + default_branch: string; + name: string; + linked_projects: { + title: string; + owner_id: string; + owner_name: string; + }[]; + }[]; +} + export interface DeployInfo { id: string; status: string; From fa5e762407e921171b05f4e9f4a4515416ab1e5b Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Tue, 8 Oct 2024 14:17:07 -0400 Subject: [PATCH 02/44] correct ellipsis --- src/deploy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deploy.ts b/src/deploy.ts index ddd704b15..d36f334df 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -215,7 +215,7 @@ class Deployer { const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; await this.apiClient.postProjectBuild(deployTarget.project.id); const spinner = this.effects.clack.spinner(); - spinner.start("Requesting deploy…"); + spinner.start("Requesting deploy"); const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; pollLoop: while (true) { if (Date.now() > pollExpiration) { From 6e614cace99a1c1e68bd08ae61956ab19c861208 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 11 Oct 2024 13:38:18 -0400 Subject: [PATCH 03/44] continuousDeployment flag persisted in deploy.json --- src/deploy.ts | 104 ++++++++++++++++++++++--------------- src/observableApiConfig.ts | 6 ++- 2 files changed, 65 insertions(+), 45 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index d36f334df..a200971a3 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -203,41 +203,35 @@ class Deployer { return deployInfo; } - private async maybeCloudBuild(deployTarget: DeployTargetInfo) { + private async cloudBuild(deployTarget: DeployTargetInfo) { if (deployTarget.create) return false; - const confirmCloudBuild = await this.effects.clack.confirm({ - message: "Do you want to build in the cloud?", - active: "Yes", - inactive: "No" - }); - if (confirmCloudBuild) { - // kick off new cloud deploy w/ link to - const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; - await this.apiClient.postProjectBuild(deployTarget.project.id); - const spinner = this.effects.clack.spinner(); - spinner.start("Requesting deploy"); - const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; - pollLoop: while (true) { - if (Date.now() > pollExpiration) { - spinner.stop("Requesting deploy timed out"); - throw new CliError("Requesting deploy failed"); - } - const {latestCreatedDeployId} = await this.apiClient.getProject({ - workspaceLogin: deployTarget.workspace.login, - projectSlug: deployTarget.project.slug - }); - if (latestCreatedDeployId !== deployTarget.project.latestCreatedDeployId) { - // TODO use observable_origin env var? - spinner.stop( - `Deploy started. Watch logs: https://observable.test:5000/projects/@${deployTarget.workspace.login}/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` - ); - break pollLoop; - } - await new Promise((resolve) => setTimeout(resolve, pollInterval)); + + // kick off new cloud deploy w/ link to deploy details + const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; + await this.apiClient.postProjectBuild(deployTarget.project.id); + const spinner = this.effects.clack.spinner(); + spinner.start("Requesting deploy"); + const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; + pollLoop: while (true) { + if (Date.now() > pollExpiration) { + spinner.stop("Requesting deploy timed out"); + throw new CliError("Requesting deploy failed"); } - return true; + const {latestCreatedDeployId} = await this.apiClient.getProject({ + workspaceLogin: deployTarget.workspace.login, + projectSlug: deployTarget.project.slug + }); + if (latestCreatedDeployId !== deployTarget.project.latestCreatedDeployId) { + spinner.stop( + `Deploy started. Watch logs: ${process.env["OBSERVABLE_ORIGIN"] ?? "https://observablehq.com/"}projects/@${ + deployTarget.workspace.login + }/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` + ); + break pollLoop; + } + await new Promise((resolve) => setTimeout(resolve, pollInterval)); } - return false; + return true; } private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { @@ -258,10 +252,16 @@ class Deployer { if (gitHub) { const repoName = formatGitUrl(gitHub[1]); const confirmLinkGitHub = await this.effects.clack.confirm({ - message: `Do you want to link to GitHub repo ${repoName}?`, + message: `Do you want to link to GitHub repository ${repoName}?`, active: "Yes", inactive: "No" }); + if (this.effects.clack.isCancel(confirmLinkGitHub) || !confirmLinkGitHub) { + throw new CliError( + "Continuous deployment is enabled in deploy.json but you cannot deploy in the cloud without a GitHub repository", + {print: true, exitCode: 0} + ); + } if (confirmLinkGitHub) { const {repositories} = await this.apiClient.getGitHubRepositories(); const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); @@ -296,16 +296,17 @@ class Deployer { private async startNewDeploy(): Promise { const deployConfig = await this.getUpdatedDeployConfig(); const deployTarget = await this.getDeployTarget(deployConfig); - const linkedGitHub = await this.maybeLinkGitHub(deployTarget); - if (linkedGitHub) { - const cloudBuild = await this.maybeCloudBuild(deployTarget); - if (cloudBuild) return true; + const deployConfig2 = await this.getUpdatedDeployConfig(); // TODO inelegant… move cd prompt to getUpdatedDeployConfig? + let deployId; + if (deployConfig2.continuousDeployment) { + await this.maybeLinkGitHub(deployTarget); + deployId = await this.cloudBuild(deployTarget); + } else { + const buildFilePaths = await this.getBuildFilePaths(); + deployId = await this.createNewDeploy(deployTarget); + await this.uploadFiles(deployId, buildFilePaths); + await this.markDeployUploaded(deployId); } - const buildFilePaths = await this.getBuildFilePaths(); - const deployId = await this.createNewDeploy(deployTarget); - - await this.uploadFiles(deployId, buildFilePaths); - await this.markDeployUploaded(deployId); return await this.pollForProcessingCompletion(deployId); } @@ -350,6 +351,8 @@ class Deployer { ); } + // TODO validate continuousDeployment + if (deployConfig.projectId && (!deployConfig.projectSlug || !deployConfig.workspaceLogin)) { const spinner = this.effects.clack.spinner(); this.effects.clack.log.warn("The `projectSlug` or `workspaceLogin` is missing from your deploy.json."); @@ -503,13 +506,28 @@ class Deployer { } } + let continuousDeployment = deployConfig.continuousDeployment; + if (continuousDeployment === null) { + continuousDeployment = !!(await this.effects.clack.confirm({ + message: wrapAnsi( + `Do you want to enable continuous deployment? ${faint( + "This builds in the cloud instead of on this machine and redeploys whenever you push to this repository." + )}`, + this.effects.outputColumns + ), + active: "Yes", + inactive: "No" + })); + } + await this.effects.setDeployConfig( this.deployOptions.config.root, this.deployOptions.deployConfigPath, { projectId: deployTarget.project.id, projectSlug: deployTarget.project.slug, - workspaceLogin: deployTarget.workspace.login + workspaceLogin: deployTarget.workspace.login, + continuousDeployment }, this.effects ); diff --git a/src/observableApiConfig.ts b/src/observableApiConfig.ts index 59470c9d6..b87f4e3d9 100644 --- a/src/observableApiConfig.ts +++ b/src/observableApiConfig.ts @@ -36,6 +36,7 @@ export interface DeployConfig { projectId?: string | null; projectSlug: string | null; workspaceLogin: string | null; + continuousDeployment: boolean | null; } export type ApiKey = @@ -87,11 +88,12 @@ export async function getDeployConfig( } } // normalize - let {projectId, projectSlug, workspaceLogin} = config ?? ({} as any); + let {projectId, projectSlug, workspaceLogin, continuousDeployment} = config ?? ({} as any); if (typeof projectId !== "string") projectId = null; if (typeof projectSlug !== "string") projectSlug = null; if (typeof workspaceLogin !== "string") workspaceLogin = null; - return {projectId, projectSlug, workspaceLogin}; + if (typeof continuousDeployment !== "boolean") continuousDeployment = null; + return {projectId, projectSlug, workspaceLogin, continuousDeployment}; } export async function setDeployConfig( From a7efd4cbed055ec79a3816ac6924aea61ee2263b Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 11 Oct 2024 13:51:40 -0400 Subject: [PATCH 04/44] poll for repo access --- src/deploy.ts | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index a200971a3..041ea5bbd 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -227,7 +227,8 @@ class Deployer { deployTarget.workspace.login }/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` ); - break pollLoop; + // break pollLoop; + return latestCreatedDeployId; } await new Promise((resolve) => setTimeout(resolve, pollInterval)); } @@ -279,8 +280,33 @@ class Deployer { } else { // repo not auth’ed; kick off web auth flow this.effects.clack.log.info( - "Authorize Observable to access this repo: https://github.com/apps/observable-data-apps-dev/installations/select_target" + "Authorize Observable to access this repository: https://github.com/apps/observable-data-apps-dev/installations/select_target" ); + const spinner = this.effects.clack.spinner(); + spinner.start("Waiting for repository to be authorized"); + const pollExpiration = Date.now() + 2 * 60_000; //DEPLOY_POLL_MAX_MS; // TODO + pollLoop: while (true) { + if (Date.now() > pollExpiration) { + spinner.stop("Waiting for repository to be authorized timed out"); + throw new CliError("Deploy failed"); + } + const {repositories} = await this.apiClient.getGitHubRepositories(); + const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); + if (authedRepo) { + spinner.stop("Repository authorized"); + await this.apiClient.postProjectEnvironment(deployTarget.project.id, { + source: { + provider: authedRepo.provider, + provider_id: authedRepo.provider_id, + url: authedRepo.url, + branch: null // TODO detect branch + } + }); + // break pollLoop; // TODO + return true; + } + await new Promise((resolve) => setTimeout(resolve, 2000)); + } } } } else { From 1077838fa5eeafe172a9983d3e3733582f582af5 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 11 Oct 2024 18:01:13 -0400 Subject: [PATCH 05/44] cleanup. remove "Do you wanna link GitHub" prompt, since you have to --- src/deploy.ts | 107 +++++++++++++++++++++----------------------------- 1 file changed, 45 insertions(+), 62 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 041ea5bbd..53833c93d 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -204,15 +204,13 @@ class Deployer { } private async cloudBuild(deployTarget: DeployTargetInfo) { - if (deployTarget.create) return false; - - // kick off new cloud deploy w/ link to deploy details + if (deployTarget.create) return false; // TODO const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; await this.apiClient.postProjectBuild(deployTarget.project.id); const spinner = this.effects.clack.spinner(); spinner.start("Requesting deploy"); const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; - pollLoop: while (true) { + while (true) { if (Date.now() > pollExpiration) { spinner.stop("Requesting deploy timed out"); throw new CliError("Requesting deploy failed"); @@ -227,12 +225,10 @@ class Deployer { deployTarget.workspace.login }/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` ); - // break pollLoop; return latestCreatedDeployId; } await new Promise((resolve) => setTimeout(resolve, pollInterval)); } - return true; } private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { @@ -241,7 +237,7 @@ class Deployer { // can do cloud build return true; } else { - // TODO Where should it look for .git? + // TODO Where should it look for .git? only supports projects at root rn… // const isGit = existsSync(this.deployOptions.config.root + "/.git"); const isGit = existsSync(".git"); if (isGit) { @@ -252,68 +248,54 @@ class Deployer { const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); if (gitHub) { const repoName = formatGitUrl(gitHub[1]); - const confirmLinkGitHub = await this.effects.clack.confirm({ - message: `Do you want to link to GitHub repository ${repoName}?`, - active: "Yes", - inactive: "No" - }); - if (this.effects.clack.isCancel(confirmLinkGitHub) || !confirmLinkGitHub) { - throw new CliError( - "Continuous deployment is enabled in deploy.json but you cannot deploy in the cloud without a GitHub repository", - {print: true, exitCode: 0} + const {repositories} = await this.apiClient.getGitHubRepositories(); + const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); + if (authedRepo) { + // authed repo found + await this.apiClient.postProjectEnvironment(deployTarget.project.id, { + source: { + provider: authedRepo.provider, + provider_id: authedRepo.provider_id, + url: authedRepo.url, + branch: null // TODO detect branch + } + }); + return true; + } else { + // repo not auth’ed; link to auth page and poll for auth + this.effects.clack.log.info( + `Authorize Observable to access the ${bold(repoName)} repository: ${link("https://github.com/apps/observable-data-apps-dev/installations/select_target")}` ); - } - if (confirmLinkGitHub) { - const {repositories} = await this.apiClient.getGitHubRepositories(); - const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); - if (authedRepo) { - // authed repo found - await this.apiClient.postProjectEnvironment(deployTarget.project.id, { - source: { - provider: authedRepo.provider, - provider_id: authedRepo.provider_id, - url: authedRepo.url, - branch: null // TODO detect branch - } - }); - return true; - } else { - // repo not auth’ed; kick off web auth flow - this.effects.clack.log.info( - "Authorize Observable to access this repository: https://github.com/apps/observable-data-apps-dev/installations/select_target" - ); - const spinner = this.effects.clack.spinner(); - spinner.start("Waiting for repository to be authorized"); - const pollExpiration = Date.now() + 2 * 60_000; //DEPLOY_POLL_MAX_MS; // TODO - pollLoop: while (true) { - if (Date.now() > pollExpiration) { - spinner.stop("Waiting for repository to be authorized timed out"); - throw new CliError("Deploy failed"); - } - const {repositories} = await this.apiClient.getGitHubRepositories(); - const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); - if (authedRepo) { - spinner.stop("Repository authorized"); - await this.apiClient.postProjectEnvironment(deployTarget.project.id, { - source: { - provider: authedRepo.provider, - provider_id: authedRepo.provider_id, - url: authedRepo.url, - branch: null // TODO detect branch - } - }); - // break pollLoop; // TODO - return true; - } - await new Promise((resolve) => setTimeout(resolve, 2000)); + const spinner = this.effects.clack.spinner(); + spinner.start("Waiting for repository to be authorized"); + const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; + while (true) { + if (Date.now() > pollExpiration) { + spinner.stop("Waiting for repository to be authorized timed out"); + throw new CliError("Deploy failed"); + } + const {repositories} = await this.apiClient.getGitHubRepositories(); + const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); + if (authedRepo) { + spinner.stop("Repository authorized"); + await this.apiClient.postProjectEnvironment(deployTarget.project.id, { + source: { + provider: authedRepo.provider, + provider_id: authedRepo.provider_id, + url: authedRepo.url, + branch: null // TODO detect branch + } + }); + return true; } + await new Promise((resolve) => setTimeout(resolve, 2000)); } } } else { - // no github remote + throw new CliError("No GitHub remote"); // TODO better error } } else { - // not a repo + throw new CliError("Not at root of a git repository"); // TODO better error } } return false; @@ -325,6 +307,7 @@ class Deployer { const deployConfig2 = await this.getUpdatedDeployConfig(); // TODO inelegant… move cd prompt to getUpdatedDeployConfig? let deployId; if (deployConfig2.continuousDeployment) { + // TODO move maybeLinkGitHub so that continuous deployment is only enabled if it succeeds await this.maybeLinkGitHub(deployTarget); deployId = await this.cloudBuild(deployTarget); } else { From 44b1e5f26dd6fbc60eda8e4b4b4aaff34fa3d39b Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Mon, 14 Oct 2024 21:06:05 -0400 Subject: [PATCH 06/44] =?UTF-8?q?various=20post-demo=20cleanup;=20maybeLin?= =?UTF-8?q?kGitHub=20even=20if=20we=E2=80=99re=20not=20enabling=20continuo?= =?UTF-8?q?usDeployment=20for=20the=20first=20time?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/deploy.ts | 76 +++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 32 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 53833c93d..0016e6e31 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -204,7 +204,9 @@ class Deployer { } private async cloudBuild(deployTarget: DeployTargetInfo) { - if (deployTarget.create) return false; // TODO + if (deployTarget.create) { + throw Error("Incorrect deployTarget state"); + } const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; await this.apiClient.postProjectBuild(deployTarget.project.id); const spinner = this.effects.clack.spinner(); @@ -225,20 +227,25 @@ class Deployer { deployTarget.workspace.login }/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` ); - return latestCreatedDeployId; + // latestCreatedDeployId is initially null for a new project, but once + // it changes to a string it can never change back; since we know it has + // changed, we assert here that it’s not null + return latestCreatedDeployId!; } await new Promise((resolve) => setTimeout(resolve, pollInterval)); } } private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { - if (!this.effects.isTty || deployTarget.create) return false; + if (deployTarget.create) { + throw Error("Incorrect deployTarget state"); + } + if (!this.effects.isTty) return false; if (deployTarget.environment.build_environment_id && deployTarget.environment.source) { // can do cloud build return true; } else { - // TODO Where should it look for .git? only supports projects at root rn… - // const isGit = existsSync(this.deployOptions.config.root + "/.git"); + // We only support cloud builds from the root directory so this ignores this.deployOptions.config.root const isGit = existsSync(".git"); if (isGit) { const remotes = (await promisify(exec)("git remote -v", {cwd: this.deployOptions.config.root})).stdout @@ -264,7 +271,9 @@ class Deployer { } else { // repo not auth’ed; link to auth page and poll for auth this.effects.clack.log.info( - `Authorize Observable to access the ${bold(repoName)} repository: ${link("https://github.com/apps/observable-data-apps-dev/installations/select_target")}` + `Authorize Observable to access the ${bold(repoName)} repository: ${link( + "https://github.com/apps/observable-data-apps-dev/installations/select_target" + )}` ); const spinner = this.effects.clack.spinner(); spinner.start("Waiting for repository to be authorized"); @@ -292,23 +301,19 @@ class Deployer { } } } else { - throw new CliError("No GitHub remote"); // TODO better error + this.effects.clack.log.error("No GitHub remote found"); } } else { - throw new CliError("Not at root of a git repository"); // TODO better error + this.effects.clack.log.error("Not at root of a git repository"); } } return false; } private async startNewDeploy(): Promise { - const deployConfig = await this.getUpdatedDeployConfig(); - const deployTarget = await this.getDeployTarget(deployConfig); - const deployConfig2 = await this.getUpdatedDeployConfig(); // TODO inelegant… move cd prompt to getUpdatedDeployConfig? - let deployId; - if (deployConfig2.continuousDeployment) { - // TODO move maybeLinkGitHub so that continuous deployment is only enabled if it succeeds - await this.maybeLinkGitHub(deployTarget); + const {deployConfig, deployTarget} = await this.getDeployTarget(await this.getUpdatedDeployConfig()); + let deployId: string | null; + if (deployConfig.continuousDeployment) { deployId = await this.cloudBuild(deployTarget); } else { const buildFilePaths = await this.getBuildFilePaths(); @@ -360,8 +365,6 @@ class Deployer { ); } - // TODO validate continuousDeployment - if (deployConfig.projectId && (!deployConfig.projectSlug || !deployConfig.workspaceLogin)) { const spinner = this.effects.clack.spinner(); this.effects.clack.log.warn("The `projectSlug` or `workspaceLogin` is missing from your deploy.json."); @@ -394,7 +397,9 @@ class Deployer { } // Get the deploy target, prompting the user as needed. - private async getDeployTarget(deployConfig: DeployConfig): Promise { + private async getDeployTarget( + deployConfig: DeployConfig + ): Promise<{deployTarget: DeployTargetInfo; deployConfig: DeployConfig}> { let deployTarget: DeployTargetInfo; if (deployConfig.workspaceLogin && deployConfig.projectSlug) { try { @@ -481,11 +486,11 @@ class Deployer { workspaceId: deployTarget.workspace.id, accessLevel: deployTarget.accessLevel }); - // TODO(toph): initial env config deployTarget = { create: false, workspace: deployTarget.workspace, project, + // TODO: In the future we may have a default environment environment: { automatic_builds_enabled: null, build_environment_id: null, @@ -515,33 +520,40 @@ class Deployer { } } - let continuousDeployment = deployConfig.continuousDeployment; + let {continuousDeployment} = deployConfig; if (continuousDeployment === null) { - continuousDeployment = !!(await this.effects.clack.confirm({ + const enable = await this.effects.clack.confirm({ message: wrapAnsi( `Do you want to enable continuous deployment? ${faint( - "This builds in the cloud instead of on this machine and redeploys whenever you push to this repository." + "This builds in the cloud and redeploys whenever you push to this repository." )}`, this.effects.outputColumns ), - active: "Yes", - inactive: "No" - })); + active: "Yes, enable and build in cloud", + inactive: "No, build locally" + }); + if (this.effects.clack.isCancel(enable)) throw new CliError("User canceled deploy", {print: false, exitCode: 0}); + continuousDeployment = enable; } + // Disables continuous deployment if there’s no env/source & we can’t link GitHub + if (continuousDeployment) continuousDeployment = await this.maybeLinkGitHub(deployTarget); + + const newDeployConfig = { + projectId: deployTarget.project.id, + projectSlug: deployTarget.project.slug, + workspaceLogin: deployTarget.workspace.login, + continuousDeployment + }; + await this.effects.setDeployConfig( this.deployOptions.config.root, this.deployOptions.deployConfigPath, - { - projectId: deployTarget.project.id, - projectSlug: deployTarget.project.slug, - workspaceLogin: deployTarget.workspace.login, - continuousDeployment - }, + newDeployConfig, this.effects ); - return deployTarget; + return {deployConfig: newDeployConfig, deployTarget}; } // Create the new deploy on the server. From b6c2a3fbd11ba620fad870cfc4a4e687a865d224 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Mon, 14 Oct 2024 22:46:16 -0400 Subject: [PATCH 07/44] better handling of the case where github is not connected at all; set primary branch in our db to current branch; add some periods to some messages where it seems appropriate for consistency --- src/deploy.ts | 30 +++++++++++++++++------------- src/observableApiClient.ts | 8 ++++++-- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 0016e6e31..12a501c4b 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -214,7 +214,7 @@ class Deployer { const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; while (true) { if (Date.now() > pollExpiration) { - spinner.stop("Requesting deploy timed out"); + spinner.stop("Requesting deploy timed out."); throw new CliError("Requesting deploy failed"); } const {latestCreatedDeployId} = await this.apiClient.getProject({ @@ -223,7 +223,7 @@ class Deployer { }); if (latestCreatedDeployId !== deployTarget.project.latestCreatedDeployId) { spinner.stop( - `Deploy started. Watch logs: ${process.env["OBSERVABLE_ORIGIN"] ?? "https://observablehq.com/"}projects/@${ + `Deploy started. Watch logs: ${process.env["OBSERVABLE_ORIGIN"] ?? "https://observablehq.com"}/projects/@${ deployTarget.workspace.login }/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` ); @@ -255,21 +255,25 @@ class Deployer { const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); if (gitHub) { const repoName = formatGitUrl(gitHub[1]); - const {repositories} = await this.apiClient.getGitHubRepositories(); - const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); + const repositories = (await this.apiClient.getGitHubRepositories())?.repositories; + const authedRepo = repositories?.find(({url}) => formatGitUrl(url) === repoName); if (authedRepo) { - // authed repo found + // Set branch to current branch + const branch = ( + await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root}) + ).stdout; await this.apiClient.postProjectEnvironment(deployTarget.project.id, { source: { provider: authedRepo.provider, provider_id: authedRepo.provider_id, url: authedRepo.url, - branch: null // TODO detect branch + branch } }); return true; } else { // repo not auth’ed; link to auth page and poll for auth + // TODO: link to internal page that bookends the flow and handles the no-oauth-token case more gracefully this.effects.clack.log.info( `Authorize Observable to access the ${bold(repoName)} repository: ${link( "https://github.com/apps/observable-data-apps-dev/installations/select_target" @@ -280,13 +284,13 @@ class Deployer { const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; while (true) { if (Date.now() > pollExpiration) { - spinner.stop("Waiting for repository to be authorized timed out"); - throw new CliError("Deploy failed"); + spinner.stop("Waiting for repository to be authorized timed out."); + throw new CliError("Repository authorization failed"); } - const {repositories} = await this.apiClient.getGitHubRepositories(); - const authedRepo = repositories.find(({url}) => formatGitUrl(url) === repoName); + const repositories = (await this.apiClient.getGitHubRepositories())?.repositories; + const authedRepo = repositories?.find(({url}) => formatGitUrl(url) === repoName); if (authedRepo) { - spinner.stop("Repository authorized"); + spinner.stop("Repository authorized."); await this.apiClient.postProjectEnvironment(deployTarget.project.id, { source: { provider: authedRepo.provider, @@ -301,10 +305,10 @@ class Deployer { } } } else { - this.effects.clack.log.error("No GitHub remote found"); + this.effects.clack.log.error("No GitHub remote found; cannot enable continuous deployment."); } } else { - this.effects.clack.log.error("Not at root of a git repository"); + this.effects.clack.log.error("Not at root of a git repository; cannot enable continuous deployment."); } } return false; diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index 990c7d5f8..f97e30e08 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -131,9 +131,13 @@ export class ObservableApiClient { return await this._fetch(url, {method: "GET"}); } - async getGitHubRepositories(): Promise { + async getGitHubRepositories(): Promise { const url = new URL("/cli/github/repositories", this._apiOrigin); - return await this._fetch(url, {method: "GET"}); + try { + return await this._fetch(url, {method: "GET"}); + } catch (err) { + return null; + } } async postProjectEnvironment(id, body): Promise { From bb790ef0af031dcb4cead1afc4ad5f60e7472874 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Tue, 15 Oct 2024 12:09:03 -0700 Subject: [PATCH 08/44] =?UTF-8?q?throw=20Error=20=E2=86=92=20throw=20new?= =?UTF-8?q?=20Error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Mike Bostock --- src/deploy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 12a501c4b..7c0fcbc61 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -205,7 +205,7 @@ class Deployer { private async cloudBuild(deployTarget: DeployTargetInfo) { if (deployTarget.create) { - throw Error("Incorrect deployTarget state"); + throw new Error("Incorrect deployTarget state"); } const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; await this.apiClient.postProjectBuild(deployTarget.project.id); @@ -238,7 +238,7 @@ class Deployer { private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { if (deployTarget.create) { - throw Error("Incorrect deployTarget state"); + throw new Error("Incorrect deployTarget state"); } if (!this.effects.isTty) return false; if (deployTarget.environment.build_environment_id && deployTarget.environment.source) { From 81281736b0278e26be9ca432584fd656ceaa8576 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Tue, 15 Oct 2024 18:04:42 -0400 Subject: [PATCH 09/44] fix existing tests --- test/deploy-test.ts | 57 ++++++++++++++++++++++++++----------- test/mocks/observableApi.ts | 29 +++++++++++++++++-- 2 files changed, 67 insertions(+), 19 deletions(-) diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 7c09a3419..c2f38bdf2 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -190,8 +190,10 @@ const TEST_OPTIONS: DeployOptions = { const DEPLOY_CONFIG: DeployConfig & {projectId: string; projectSlug: string; workspaceLogin: string} = { projectId: "project123", projectSlug: "bi", - workspaceLogin: "mock-user-ws" + workspaceLogin: "mock-user-ws", + continuousDeployment: false }; +const DEFAULT_ENVIRONMENT = {automatic_builds_enabled: null, build_environment_id: null, source: null}; describe("deploy", () => { before(() => setCurrentDate(new Date("2024-01-10T16:00:00"))); @@ -203,6 +205,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -293,6 +296,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject({...DEPLOY_CONFIG, title: oldTitle}) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -347,6 +351,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(deployConfig) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: deployConfig.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -569,9 +574,8 @@ describe("deploy", () => { const deployId = "deploy456"; getCurrentObservableApi() .handleGetCurrentUser() - .handleGetProject({ - ...DEPLOY_CONFIG - }) + .handleGetProject({...DEPLOY_CONFIG}) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId, status: 500}) .start(); const effects = new MockDeployEffects({deployConfig: DEPLOY_CONFIG}); @@ -596,6 +600,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .handlePostDeployManifest({deployId, files: [{deployId, path: "index.html", action: "upload"}]}) .handlePostDeployFile({deployId, status: 500}) @@ -620,6 +625,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId, status: 500}) @@ -728,6 +734,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({ @@ -761,10 +768,8 @@ describe("deploy", () => { const deployId = "deployId"; getCurrentObservableApi() .handleGetCurrentUser() - .handleGetProject({ - ...DEPLOY_CONFIG, - projectId: newProjectId - }) + .handleGetProject({...DEPLOY_CONFIG, projectId: newProjectId}) + .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: newProjectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -782,10 +787,8 @@ describe("deploy", () => { const oldDeployConfig = {...DEPLOY_CONFIG, projectId: "oldProjectId"}; getCurrentObservableApi() .handleGetCurrentUser() - .handleGetProject({ - ...DEPLOY_CONFIG, - projectId: newProjectId - }) + .handleGetProject({...DEPLOY_CONFIG, projectId: newProjectId}) + .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({deployConfig: oldDeployConfig, isTty: true}); effects.clack.inputs.push(false); // State doesn't match do you want to continue deploying? @@ -808,6 +811,7 @@ describe("deploy", () => { ...DEPLOY_CONFIG, projectId: newProjectId }) + .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({deployConfig: oldDeployConfig, isTty: false, debug: true}); try { @@ -829,6 +833,7 @@ describe("deploy", () => { ...DEPLOY_CONFIG, projectId: newProjectId }) + .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({deployConfig, isTty: true}); effects.clack.inputs.push(false); @@ -849,6 +854,7 @@ describe("deploy", () => { .handlePostAuthRequestPoll("accepted") .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -870,6 +876,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -894,7 +901,11 @@ describe("deploy", () => { force: null, config: {...TEST_OPTIONS.config, output: "test/output/does-not-exist"} } satisfies DeployOptions; - getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) + .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-09"), @@ -909,7 +920,11 @@ describe("deploy", () => { ...TEST_OPTIONS, force: null } satisfies DeployOptions; - getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) + .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-09"), @@ -927,7 +942,11 @@ describe("deploy", () => { ...TEST_OPTIONS, force: null } satisfies DeployOptions; - getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) + .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-11"), @@ -945,7 +964,11 @@ describe("deploy", () => { ...TEST_OPTIONS, force: "build" } satisfies DeployOptions; - getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) + .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-09"), @@ -971,6 +994,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -1002,6 +1026,7 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) + .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectFileUpload({deployId, path: "index.html", action: "upload"}) .expectFileUpload({deployId, path: "_observablehq/client.00000001.js", action: "skip"}) diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts index 65593636f..d497afdb7 100644 --- a/test/mocks/observableApi.ts +++ b/test/mocks/observableApi.ts @@ -4,6 +4,7 @@ import PendingInterceptorsFormatter from "undici/lib/mock/pending-interceptors-f import type {BuildManifest} from "../../src/build.js"; import type { GetCurrentUserResponse, + GetProjectEnvironmentResponse, GetProjectResponse, PaginatedList, PostAuthRequestPollResponse, @@ -166,7 +167,8 @@ class ObservableApiMock { slug: projectSlug, title, creator: {id: "user-id", login: "user-login"}, - owner: {id: "workspace-id", login: "workspace-login"} + owner: {id: "workspace-id", login: "workspace-login"}, + latestCreatedDeployId: null } satisfies GetProjectResponse) : emptyErrorBody; const headers = authorizationHeader(status !== 401 && status !== 403); @@ -203,7 +205,8 @@ class ObservableApiMock { slug, title: "Mock Project", owner, - creator + creator, + latestCreatedDeployId: null } satisfies GetProjectResponse) : emptyErrorBody; const headers = authorizationHeader(status !== 403); @@ -235,7 +238,8 @@ class ObservableApiMock { creator, owner, title: p.title ?? "Mock Title", - accessLevel: p.accessLevel ?? "private" + accessLevel: p.accessLevel ?? "private", + latestCreatedDeployId: null })) } satisfies PaginatedList) : emptyErrorBody; @@ -263,6 +267,25 @@ class ObservableApiMock { return this; } + handleGetProjectEnvironment({ + projectId, + environment, + status = 200 + }: { + projectId: string; + environment?: GetProjectEnvironmentResponse; + status?: number; + }): ObservableApiMock { + const response = status == 200 ? JSON.stringify(environment) : emptyErrorBody; + const headers = authorizationHeader(status !== 403); + this.addHandler((pool) => + pool + .intercept({path: `/cli/project/${projectId}/environment`, method: "GET", headers: headersMatcher(headers)}) + .reply(status, response, {headers: {"content-type": "application/json"}}) + ); + return this; + } + expectStandardFiles(options: Omit) { return this.expectFileUpload({...options, path: "index.html"}) .expectFileUpload({...options, path: "_observablehq/client.00000001.js"}) From 56e4d146e30c453383f5edb06ed5848144bfe749 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Tue, 22 Oct 2024 16:52:42 -0400 Subject: [PATCH 10/44] start on docs --- docs/deploying.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/deploying.md b/docs/deploying.md index 6beb62651..7113a0811 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -36,9 +36,11 @@ npm run deploy -- --help -## Automated deploys +## Continuous deployment -After deploying an app manually at least once, Observable can handle subsequent deploys for you automatically. You can automate deploys both [on commit](https://observablehq.com/documentation/data-apps/github) (whenever you push a new commit to your project’s default branch) and [on schedule](https://observablehq.com/documentation/data-apps/schedules) (such as daily or weekly). + + +You can connect your app to Observable to handle deploys automatically. You can automate deploys both [on commit](https://observablehq.com/documentation/data-apps/github) (whenever you push a new commit to your project’s default branch) and [on schedule](https://observablehq.com/documentation/data-apps/schedules) (such as daily or weekly). Automatic deploys — also called _continuous deployment_ or _CD_ — ensure that your data is always up to date, and that any changes you make to your app are immediately reflected in the deployed version. @@ -88,7 +90,7 @@ To create an API key: 1. Open the [API Key settings](https://observablehq.com/select-workspace?next=api-keys-settings) for your Observable workspace. 2. Click **New API Key**. -3. Check the **Deploy new versions of projects** checkbox. +3. Check the **Deploy new versions of data apps** checkbox. 4. Give your key a description, such as “Deploy via GitHub Actions”. 5. Click **Create API Key**. @@ -147,7 +149,8 @@ The contents of the deploy config file look like this: { "projectId": "0123456789abcdef", "projectSlug": "hello-framework", - "workspaceLogin": "acme" + "workspaceLogin": "acme", + "continuousDeployment": true } ``` From 9e234a2d9135173b2cb907fd24612200bbb30c37 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 25 Oct 2024 15:03:18 -0700 Subject: [PATCH 11/44] use new singular endpoint to see if repo is authed --- src/deploy.ts | 6 ++---- src/observableApiClient.ts | 34 ++++++++++++++-------------------- 2 files changed, 16 insertions(+), 24 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 7c0fcbc61..fa4d656ce 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -255,8 +255,7 @@ class Deployer { const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); if (gitHub) { const repoName = formatGitUrl(gitHub[1]); - const repositories = (await this.apiClient.getGitHubRepositories())?.repositories; - const authedRepo = repositories?.find(({url}) => formatGitUrl(url) === repoName); + const authedRepo = (await this.apiClient.getGitHubRepository(repoName)); if (authedRepo) { // Set branch to current branch const branch = ( @@ -287,8 +286,7 @@ class Deployer { spinner.stop("Waiting for repository to be authorized timed out."); throw new CliError("Repository authorization failed"); } - const repositories = (await this.apiClient.getGitHubRepositories())?.repositories; - const authedRepo = repositories?.find(({url}) => formatGitUrl(url) === repoName); + const authedRepo = (await this.apiClient.getGitHubRepository(repoName)); if (authedRepo) { spinner.stop("Repository authorized."); await this.apiClient.postProjectEnvironment(deployTarget.project.id, { diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index f97e30e08..46c113ee4 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -131,10 +131,11 @@ export class ObservableApiClient { return await this._fetch(url, {method: "GET"}); } - async getGitHubRepositories(): Promise { - const url = new URL("/cli/github/repositories", this._apiOrigin); + async getGitHubRepository(repoName): Promise { + const [owner, repo] = repoName.split("/"); + const url = new URL(`/cli/github/repository?owner=${owner}&repo=${repo}`, this._apiOrigin); try { - return await this._fetch(url, {method: "GET"}); + return await this._fetch(url, {method: "GET"}); } catch (err) { return null; } @@ -309,23 +310,16 @@ export interface GetProjectEnvironmentResponse { }; } -export interface GetGitHubRepositoriesResponse { - installations: { - id: number; - login: string | null; - name: string | null; - }[]; - repositories: { - provider: "github"; - provider_id: string; - url: string; - default_branch: string; - name: string; - linked_projects: { - title: string; - owner_id: string; - owner_name: string; - }[]; +export interface GetGitHubRepositoryResponse { + provider: "github"; + provider_id: string; + url: string; + default_branch: string; + name: string; + linked_projects: { + title: string; + owner_id: string; + owner_name: string; }[]; } From c9effe03886da12fe679359d50d87d86ade6c91d Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 25 Oct 2024 18:34:29 -0700 Subject: [PATCH 12/44] clean up logic, more dry --- src/deploy.ts | 50 +++++++++++++++++++------------------------------- 1 file changed, 19 insertions(+), 31 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index fa4d656ce..d538e9dc0 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -255,22 +255,12 @@ class Deployer { const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); if (gitHub) { const repoName = formatGitUrl(gitHub[1]); - const authedRepo = (await this.apiClient.getGitHubRepository(repoName)); - if (authedRepo) { - // Set branch to current branch - const branch = ( - await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root}) - ).stdout; - await this.apiClient.postProjectEnvironment(deployTarget.project.id, { - source: { - provider: authedRepo.provider, - provider_id: authedRepo.provider_id, - url: authedRepo.url, - branch - } - }); - return true; - } else { + // Get current branch + const branch = ( + await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root}) + ).stdout; + let authedRepo = (await this.apiClient.getGitHubRepository(repoName)); + if (!authedRepo) { // repo not auth’ed; link to auth page and poll for auth // TODO: link to internal page that bookends the flow and handles the no-oauth-token case more gracefully this.effects.clack.log.info( @@ -281,27 +271,25 @@ class Deployer { const spinner = this.effects.clack.spinner(); spinner.start("Waiting for repository to be authorized"); const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; - while (true) { + while (!authedRepo) { + await new Promise((resolve) => setTimeout(resolve, 2000)); if (Date.now() > pollExpiration) { spinner.stop("Waiting for repository to be authorized timed out."); throw new CliError("Repository authorization failed"); } - const authedRepo = (await this.apiClient.getGitHubRepository(repoName)); - if (authedRepo) { - spinner.stop("Repository authorized."); - await this.apiClient.postProjectEnvironment(deployTarget.project.id, { - source: { - provider: authedRepo.provider, - provider_id: authedRepo.provider_id, - url: authedRepo.url, - branch: null // TODO detect branch - } - }); - return true; - } - await new Promise((resolve) => setTimeout(resolve, 2000)); + authedRepo = (await this.apiClient.getGitHubRepository(repoName)); + if (authedRepo) spinner.stop("Repository authorized."); } } + await this.apiClient.postProjectEnvironment(deployTarget.project.id, { + source: { + provider: authedRepo.provider, + provider_id: authedRepo.provider_id, + url: authedRepo.url, + branch + } + }); + return true; } else { this.effects.clack.log.error("No GitHub remote found; cannot enable continuous deployment."); } From 8c75f5284e476983a2db7a8797277b56c4c722de Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Tue, 29 Oct 2024 14:42:31 -0700 Subject: [PATCH 13/44] rm getProjectEnvironment calls --- src/deploy.ts | 26 ++++++-------------------- src/observableApiClient.ts | 19 +++++++++++-------- test/deploy-test.ts | 20 -------------------- test/mocks/observableApi.ts | 35 ++++++++++++----------------------- 4 files changed, 29 insertions(+), 71 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index d538e9dc0..e024a63d5 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -23,7 +23,6 @@ import type { DeployManifestFile, GetCurrentUserResponse, GetDeployResponse, - GetProjectEnvironmentResponse, GetProjectResponse, WorkspaceResponse } from "./observableApiClient.js"; @@ -94,7 +93,6 @@ type DeployTargetInfo = create: false; workspace: {id: string; login: string}; project: GetProjectResponse; - environment: GetProjectEnvironmentResponse; }; /** Deploy a project to Observable */ @@ -241,7 +239,7 @@ class Deployer { throw new Error("Incorrect deployTarget state"); } if (!this.effects.isTty) return false; - if (deployTarget.environment.build_environment_id && deployTarget.environment.source) { + if (deployTarget.project.build_environment_id && deployTarget.project.source) { // can do cloud build return true; } else { @@ -259,7 +257,7 @@ class Deployer { const branch = ( await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root}) ).stdout; - let authedRepo = (await this.apiClient.getGitHubRepository(repoName)); + let authedRepo = await this.apiClient.getGitHubRepository(repoName); if (!authedRepo) { // repo not auth’ed; link to auth page and poll for auth // TODO: link to internal page that bookends the flow and handles the no-oauth-token case more gracefully @@ -277,7 +275,7 @@ class Deployer { spinner.stop("Waiting for repository to be authorized timed out."); throw new CliError("Repository authorization failed"); } - authedRepo = (await this.apiClient.getGitHubRepository(repoName)); + authedRepo = await this.apiClient.getGitHubRepository(repoName); if (authedRepo) spinner.stop("Repository authorized."); } } @@ -397,8 +395,7 @@ class Deployer { workspaceLogin: deployConfig.workspaceLogin, projectSlug: deployConfig.projectSlug }); - const environment = await this.apiClient.getProjectEnvironment({id: project.id}); - deployTarget = {create: false, workspace: project.owner, project, environment}; + deployTarget = {create: false, workspace: project.owner, project}; } catch (error) { if (!isHttpError(error) || error.statusCode !== 404) { throw error; @@ -479,13 +476,7 @@ class Deployer { deployTarget = { create: false, workspace: deployTarget.workspace, - project, - // TODO: In the future we may have a default environment - environment: { - automatic_builds_enabled: null, - build_environment_id: null, - source: null - } + project }; } catch (error) { if (isApiError(error) && error.details.errors.some((e) => e.code === "TOO_MANY_PROJECTS")) { @@ -908,12 +899,7 @@ export async function promptDeployTarget( return { create: false, workspace, - project: existingProjects.find((p) => p.slug === chosenProject)!, - environment: { - automatic_builds_enabled: null, - build_environment_id: null, - source: null - } + project: existingProjects.find((p) => p.slug === chosenProject)! }; } } else { diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index 46c113ee4..727299f73 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -126,11 +126,6 @@ export class ObservableApiClient { return await this._fetch(url, {method: "GET"}); } - async getProjectEnvironment({id}: {id: string}): Promise { - const url = new URL(`/cli/project/${id}/environment`, this._apiOrigin); - return await this._fetch(url, {method: "GET"}); - } - async getGitHubRepository(repoName): Promise { const [owner, repo] = repoName.split("/"); const url = new URL(`/cli/github/repository?owner=${owner}&repo=${repo}`, this._apiOrigin); @@ -141,9 +136,9 @@ export class ObservableApiClient { } } - async postProjectEnvironment(id, body): Promise { + async postProjectEnvironment(id, body): Promise { const url = new URL(`/cli/project/${id}/environment`, this._apiOrigin); - return await this._fetch(url, { + return await this._fetch(url, { method: "POST", headers: {"Content-Type": "application/json"}, body: JSON.stringify(body) @@ -295,11 +290,19 @@ export interface GetProjectResponse { owner: {id: string; login: string}; creator: {id: string; login: string}; latestCreatedDeployId: string | null; + automatic_builds_enabled: boolean | null; + build_environment_id: string | null; + source: null | { + provider: string; + provider_id: string; + url: string; + branch: string | null; + }; // Available fields that we don't use // servingRoot: string | null; } -export interface GetProjectEnvironmentResponse { +export interface PostProjectEnvironmentResponse { automatic_builds_enabled: boolean | null; build_environment_id: string | null; source: null | { diff --git a/test/deploy-test.ts b/test/deploy-test.ts index c2f38bdf2..76e95c31a 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -193,7 +193,6 @@ const DEPLOY_CONFIG: DeployConfig & {projectId: string; projectSlug: string; wor workspaceLogin: "mock-user-ws", continuousDeployment: false }; -const DEFAULT_ENVIRONMENT = {automatic_builds_enabled: null, build_environment_id: null, source: null}; describe("deploy", () => { before(() => setCurrentDate(new Date("2024-01-10T16:00:00"))); @@ -205,7 +204,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -296,7 +294,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject({...DEPLOY_CONFIG, title: oldTitle}) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -351,7 +348,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(deployConfig) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: deployConfig.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -575,7 +571,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject({...DEPLOY_CONFIG}) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId, status: 500}) .start(); const effects = new MockDeployEffects({deployConfig: DEPLOY_CONFIG}); @@ -600,7 +595,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .handlePostDeployManifest({deployId, files: [{deployId, path: "index.html", action: "upload"}]}) .handlePostDeployFile({deployId, status: 500}) @@ -625,7 +619,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId, status: 500}) @@ -734,7 +727,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({ @@ -769,7 +761,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject({...DEPLOY_CONFIG, projectId: newProjectId}) - .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: newProjectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -788,7 +779,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject({...DEPLOY_CONFIG, projectId: newProjectId}) - .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({deployConfig: oldDeployConfig, isTty: true}); effects.clack.inputs.push(false); // State doesn't match do you want to continue deploying? @@ -811,7 +801,6 @@ describe("deploy", () => { ...DEPLOY_CONFIG, projectId: newProjectId }) - .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({deployConfig: oldDeployConfig, isTty: false, debug: true}); try { @@ -833,7 +822,6 @@ describe("deploy", () => { ...DEPLOY_CONFIG, projectId: newProjectId }) - .handleGetProjectEnvironment({projectId: newProjectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({deployConfig, isTty: true}); effects.clack.inputs.push(false); @@ -854,7 +842,6 @@ describe("deploy", () => { .handlePostAuthRequestPoll("accepted") .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -876,7 +863,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -904,7 +890,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, @@ -923,7 +908,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, @@ -945,7 +929,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, @@ -967,7 +950,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, @@ -994,7 +976,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectStandardFiles({deployId}) .handlePostDeployUploaded({deployId}) @@ -1026,7 +1007,6 @@ describe("deploy", () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetProject(DEPLOY_CONFIG) - .handleGetProjectEnvironment({projectId: DEPLOY_CONFIG.projectId, environment: DEFAULT_ENVIRONMENT}) .handlePostDeploy({projectId: DEPLOY_CONFIG.projectId, deployId}) .expectFileUpload({deployId, path: "index.html", action: "upload"}) .expectFileUpload({deployId, path: "_observablehq/client.00000001.js", action: "skip"}) diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts index d497afdb7..42071bc0a 100644 --- a/test/mocks/observableApi.ts +++ b/test/mocks/observableApi.ts @@ -4,7 +4,6 @@ import PendingInterceptorsFormatter from "undici/lib/mock/pending-interceptors-f import type {BuildManifest} from "../../src/build.js"; import type { GetCurrentUserResponse, - GetProjectEnvironmentResponse, GetProjectResponse, PaginatedList, PostAuthRequestPollResponse, @@ -168,7 +167,10 @@ class ObservableApiMock { title, creator: {id: "user-id", login: "user-login"}, owner: {id: "workspace-id", login: "workspace-login"}, - latestCreatedDeployId: null + latestCreatedDeployId: null, + automatic_builds_enabled: null, + build_environment_id: null, + source: null } satisfies GetProjectResponse) : emptyErrorBody; const headers = authorizationHeader(status !== 401 && status !== 403); @@ -206,7 +208,10 @@ class ObservableApiMock { title: "Mock Project", owner, creator, - latestCreatedDeployId: null + latestCreatedDeployId: null, + automatic_builds_enabled: null, + build_environment_id: null, + source: null } satisfies GetProjectResponse) : emptyErrorBody; const headers = authorizationHeader(status !== 403); @@ -239,7 +244,10 @@ class ObservableApiMock { owner, title: p.title ?? "Mock Title", accessLevel: p.accessLevel ?? "private", - latestCreatedDeployId: null + latestCreatedDeployId: null, + automatic_builds_enabled: null, + build_environment_id: null, + source: null })) } satisfies PaginatedList) : emptyErrorBody; @@ -267,25 +275,6 @@ class ObservableApiMock { return this; } - handleGetProjectEnvironment({ - projectId, - environment, - status = 200 - }: { - projectId: string; - environment?: GetProjectEnvironmentResponse; - status?: number; - }): ObservableApiMock { - const response = status == 200 ? JSON.stringify(environment) : emptyErrorBody; - const headers = authorizationHeader(status !== 403); - this.addHandler((pool) => - pool - .intercept({path: `/cli/project/${projectId}/environment`, method: "GET", headers: headersMatcher(headers)}) - .reply(status, response, {headers: {"content-type": "application/json"}}) - ); - return this; - } - expectStandardFiles(options: Omit) { return this.expectFileUpload({...options, path: "index.html"}) .expectFileUpload({...options, path: "_observablehq/client.00000001.js"}) From 75cbf50de35993ca834118897ba8e6ae500efbd5 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Wed, 30 Oct 2024 01:11:24 -0700 Subject: [PATCH 14/44] link to internal interstitial screen instead of directly to github --- src/deploy.ts | 18 +++++++++++------- src/observableApiClient.ts | 5 ++--- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index e024a63d5..639b69545 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -18,7 +18,7 @@ import {visitFiles} from "./files.js"; import type {Logger} from "./logger.js"; import type {AuthEffects} from "./observableApiAuth.js"; import {defaultEffects as defaultAuthEffects, formatUser, loginInner, validWorkspaces} from "./observableApiAuth.js"; -import {ObservableApiClient} from "./observableApiClient.js"; +import {ObservableApiClient, getObservableUiOrigin} from "./observableApiClient.js"; import type { DeployManifestFile, GetCurrentUserResponse, @@ -36,6 +36,8 @@ const DEPLOY_POLL_MAX_MS = 1000 * 60 * 5; const DEPLOY_POLL_INTERVAL_MS = 1000 * 5; const BUILD_AGE_WARNING_MS = 1000 * 60 * 5; +const OBSERVABLE_UI_ORIGIN = getObservableUiOrigin(); + export function formatGitUrl(url: string) { return new URL(url).pathname.slice(1).replace(/\.git$/, ""); } @@ -252,18 +254,20 @@ class Deployer { .map((d) => d.split(/\s/g)); const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); if (gitHub) { - const repoName = formatGitUrl(gitHub[1]); + const [ownerName, repoName] = formatGitUrl(gitHub[1]).split("/"); // Get current branch const branch = ( await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root}) ).stdout; - let authedRepo = await this.apiClient.getGitHubRepository(repoName); + let authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); if (!authedRepo) { - // repo not auth’ed; link to auth page and poll for auth - // TODO: link to internal page that bookends the flow and handles the no-oauth-token case more gracefully + // Repo is not authorized; link to auth page and poll for auth + const authUrl = new URL("/auth-github", OBSERVABLE_UI_ORIGIN); + authUrl.searchParams.set("owner", ownerName); + authUrl.searchParams.set("repo", repoName); this.effects.clack.log.info( `Authorize Observable to access the ${bold(repoName)} repository: ${link( - "https://github.com/apps/observable-data-apps-dev/installations/select_target" + authUrl )}` ); const spinner = this.effects.clack.spinner(); @@ -275,7 +279,7 @@ class Deployer { spinner.stop("Waiting for repository to be authorized timed out."); throw new CliError("Repository authorization failed"); } - authedRepo = await this.apiClient.getGitHubRepository(repoName); + authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); if (authedRepo) spinner.stop("Repository authorized."); } } diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index 727299f73..2f620b4ab 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -126,9 +126,8 @@ export class ObservableApiClient { return await this._fetch(url, {method: "GET"}); } - async getGitHubRepository(repoName): Promise { - const [owner, repo] = repoName.split("/"); - const url = new URL(`/cli/github/repository?owner=${owner}&repo=${repo}`, this._apiOrigin); + async getGitHubRepository(ownerName, repoName): Promise { + const url = new URL(`/cli/github/repository?owner=${ownerName}&repo=${repoName}`, this._apiOrigin); try { return await this._fetch(url, {method: "GET"}); } catch (err) { From e80786b17f61d2cc2cd4b536a73d9c66c33de451 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Wed, 30 Oct 2024 16:31:01 -0700 Subject: [PATCH 15/44] slightly more human-readable error message --- src/deploy.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 639b69545..da95e441a 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -205,7 +205,7 @@ class Deployer { private async cloudBuild(deployTarget: DeployTargetInfo) { if (deployTarget.create) { - throw new Error("Incorrect deployTarget state"); + throw new Error("Incorrect deploy target state"); } const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; await this.apiClient.postProjectBuild(deployTarget.project.id); @@ -238,7 +238,7 @@ class Deployer { private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { if (deployTarget.create) { - throw new Error("Incorrect deployTarget state"); + throw new Error("Incorrect deploy target state"); } if (!this.effects.isTty) return false; if (deployTarget.project.build_environment_id && deployTarget.project.source) { From ea676b158d93ac7c028f5bc2a59baed1ee540cb6 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Wed, 30 Oct 2024 16:43:09 -0700 Subject: [PATCH 16/44] dont change continuousDeployment setting as a side effect of github link failing --- src/deploy.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/deploy.ts b/src/deploy.ts index da95e441a..39ac1d496 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -253,6 +253,7 @@ class Deployer { .filter((d) => d) .map((d) => d.split(/\s/g)); const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); + // TODO: validate "Your branch is up to date" & "nothing to commit, working tree clean" if (gitHub) { const [ownerName, repoName] = formatGitUrl(gitHub[1]).split("/"); // Get current branch @@ -522,7 +523,7 @@ class Deployer { } // Disables continuous deployment if there’s no env/source & we can’t link GitHub - if (continuousDeployment) continuousDeployment = await this.maybeLinkGitHub(deployTarget); + if (continuousDeployment) await this.maybeLinkGitHub(deployTarget); const newDeployConfig = { projectId: deployTarget.project.id, From 06b52f73d5c347baf85101a323422ae4f7642575 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Wed, 30 Oct 2024 17:25:23 -0700 Subject: [PATCH 17/44] flatten structure of maybeLinkGitHub with early returns --- src/deploy.ts | 106 +++++++++++++++++++++++--------------------------- 1 file changed, 49 insertions(+), 57 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 39ac1d496..9641b466f 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -240,67 +240,59 @@ class Deployer { if (deployTarget.create) { throw new Error("Incorrect deploy target state"); } + // We only support cloud builds from the root directory so this ignores this.deployOptions.config.root + const isGit = existsSync(".git"); + if (!isGit) throw new CliError("Not at root of a git repository; cannot enable continuous deployment."); + const remotes = (await promisify(exec)("git remote -v", {cwd: this.deployOptions.config.root})).stdout + .split("\n") + .filter((d) => d) + .map((d) => d.split(/\s/g)); + const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); + if (!gitHub) throw new CliError("No GitHub remote found; cannot enable continuous deployment."); + // TODO: validate "Your branch is up to date" & "nothing to commit, working tree clean" + + // TODO allow setting this from CLI? + if (!deployTarget.project.build_environment_id) throw new CliError("No build environment configured."); + + // can do cloud build + // TODO: validate local/remote refs match & we can access repo + if (deployTarget.project.source) return true; + + // Interactively try to link repository if (!this.effects.isTty) return false; - if (deployTarget.project.build_environment_id && deployTarget.project.source) { - // can do cloud build - return true; - } else { - // We only support cloud builds from the root directory so this ignores this.deployOptions.config.root - const isGit = existsSync(".git"); - if (isGit) { - const remotes = (await promisify(exec)("git remote -v", {cwd: this.deployOptions.config.root})).stdout - .split("\n") - .filter((d) => d) - .map((d) => d.split(/\s/g)); - const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); - // TODO: validate "Your branch is up to date" & "nothing to commit, working tree clean" - if (gitHub) { - const [ownerName, repoName] = formatGitUrl(gitHub[1]).split("/"); - // Get current branch - const branch = ( - await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root}) - ).stdout; - let authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); - if (!authedRepo) { - // Repo is not authorized; link to auth page and poll for auth - const authUrl = new URL("/auth-github", OBSERVABLE_UI_ORIGIN); - authUrl.searchParams.set("owner", ownerName); - authUrl.searchParams.set("repo", repoName); - this.effects.clack.log.info( - `Authorize Observable to access the ${bold(repoName)} repository: ${link( - authUrl - )}` - ); - const spinner = this.effects.clack.spinner(); - spinner.start("Waiting for repository to be authorized"); - const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; - while (!authedRepo) { - await new Promise((resolve) => setTimeout(resolve, 2000)); - if (Date.now() > pollExpiration) { - spinner.stop("Waiting for repository to be authorized timed out."); - throw new CliError("Repository authorization failed"); - } - authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); - if (authedRepo) spinner.stop("Repository authorized."); - } - } - await this.apiClient.postProjectEnvironment(deployTarget.project.id, { - source: { - provider: authedRepo.provider, - provider_id: authedRepo.provider_id, - url: authedRepo.url, - branch - } - }); - return true; - } else { - this.effects.clack.log.error("No GitHub remote found; cannot enable continuous deployment."); + const [ownerName, repoName] = formatGitUrl(gitHub[1]).split("/"); + // Get current branch + const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root})) + .stdout; + let authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); + if (!authedRepo) { + // Repo is not authorized; link to auth page and poll for auth + const authUrl = new URL("/auth-github", OBSERVABLE_UI_ORIGIN); + authUrl.searchParams.set("owner", ownerName); + authUrl.searchParams.set("repo", repoName); + this.effects.clack.log.info(`Authorize Observable to access the ${bold(repoName)} repository: ${link(authUrl)}`); + const spinner = this.effects.clack.spinner(); + spinner.start("Waiting for repository to be authorized"); + const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; + while (!authedRepo) { + await new Promise((resolve) => setTimeout(resolve, 2000)); + if (Date.now() > pollExpiration) { + spinner.stop("Waiting for repository to be authorized timed out."); + throw new CliError("Repository authorization failed"); } - } else { - this.effects.clack.log.error("Not at root of a git repository; cannot enable continuous deployment."); + authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); + if (authedRepo) spinner.stop("Repository authorized."); } } - return false; + const response = await this.apiClient.postProjectEnvironment(deployTarget.project.id, { + source: { + provider: authedRepo.provider, + provider_id: authedRepo.provider_id, + url: authedRepo.url, + branch + } + }); + return !!response; } private async startNewDeploy(): Promise { From a7ba250b78b2472c6f85c1e088f46c9e728e9486 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Thu, 31 Oct 2024 16:11:28 -0700 Subject: [PATCH 18/44] rename maybeLinkGitHub: boolean to validateGitHubLink: void, more error throwing, check local repo against configured remote repo, just repo and branch, not yet refs --- src/deploy.ts | 95 ++++++++++++++++++++++++++++---------- src/observableApiClient.ts | 13 +++++- 2 files changed, 81 insertions(+), 27 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 9641b466f..66c8f17ea 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -42,6 +42,13 @@ export function formatGitUrl(url: string) { return new URL(url).pathname.slice(1).replace(/\.git$/, ""); } +function settingsUrl(deployTarget: DeployTargetInfo) { + if (deployTarget.create) { + throw new Error("Incorrect deploy target state"); + } + return `${OBSERVABLE_UI_ORIGIN}projects/@${deployTarget.workspace.login}/${deployTarget.project.slug}`; +} + export interface DeployOptions { config: Config; deployConfigPath: string | undefined; @@ -223,9 +230,7 @@ class Deployer { }); if (latestCreatedDeployId !== deployTarget.project.latestCreatedDeployId) { spinner.stop( - `Deploy started. Watch logs: ${process.env["OBSERVABLE_ORIGIN"] ?? "https://observablehq.com"}/projects/@${ - deployTarget.workspace.login - }/${deployTarget.project.slug}/deploys/${latestCreatedDeployId}` + `Deploy started. Watch logs: ${link(`${settingsUrl(deployTarget)}/deploys/${latestCreatedDeployId}`)}` ); // latestCreatedDeployId is initially null for a new project, but once // it changes to a string it can never change back; since we know it has @@ -236,63 +241,104 @@ class Deployer { } } - private async maybeLinkGitHub(deployTarget: DeployTargetInfo): Promise { + // Throws error if local and remote GitHub repos don’t match or are invalid + private async validateGitHubLink(deployTarget: DeployTargetInfo): Promise { if (deployTarget.create) { throw new Error("Incorrect deploy target state"); } // We only support cloud builds from the root directory so this ignores this.deployOptions.config.root const isGit = existsSync(".git"); - if (!isGit) throw new CliError("Not at root of a git repository; cannot enable continuous deployment."); + if (!isGit) throw new CliError("Not at root of a git repository."); const remotes = (await promisify(exec)("git remote -v", {cwd: this.deployOptions.config.root})).stdout .split("\n") .filter((d) => d) .map((d) => d.split(/\s/g)); const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); - if (!gitHub) throw new CliError("No GitHub remote found; cannot enable continuous deployment."); + if (!gitHub) throw new CliError("No GitHub remote found."); // TODO: validate "Your branch is up to date" & "nothing to commit, working tree clean" - // TODO allow setting this from CLI? if (!deployTarget.project.build_environment_id) throw new CliError("No build environment configured."); + // TODO: allow setting build environment from CLI - // can do cloud build - // TODO: validate local/remote refs match & we can access repo - if (deployTarget.project.source) return true; - - // Interactively try to link repository - if (!this.effects.isTty) return false; const [ownerName, repoName] = formatGitUrl(gitHub[1]).split("/"); - // Get current branch const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root})) .stdout; - let authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); - if (!authedRepo) { + + let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); + + // If a source repository has already been configured, check that it’s + // accessible and matches the local repository and branch + if (deployTarget.project.source) { + if (localRepo && deployTarget.project.source.provider_id !== localRepo.provider_id) { + throw new CliError( + `Configured repository does not match local repository; check build settings on ${link( + `${settingsUrl(deployTarget)}/settings` + )}` + ); + } + if (localRepo && deployTarget.project.source.branch !== branch) { + throw new CliError( + `Configured branch does not match local branch; check build settings on ${link( + `${settingsUrl(deployTarget)}/settings` + )}` + ); + } + // TODO: validate local/remote refs match + const remoteAuthedRepo = await this.apiClient.getGitHubRepository({ + providerId: deployTarget.project.source.provider_id + }); + if (!remoteAuthedRepo) { + console.log(deployTarget.project.source.provider_id, remoteAuthedRepo); + throw new CliError( + `Cannot access configured repository; check build settings on ${link( + `${settingsUrl(deployTarget)}/settings` + )}` + ); + } + + // Configured repo is OK; proceed + return; + } + + if (!localRepo) { + if (!this.effects.isTty) + throw new CliError( + "Cannot access repository for continuous deployment and cannot request access in non-interactive mode" + ); + // Repo is not authorized; link to auth page and poll for auth const authUrl = new URL("/auth-github", OBSERVABLE_UI_ORIGIN); authUrl.searchParams.set("owner", ownerName); authUrl.searchParams.set("repo", repoName); this.effects.clack.log.info(`Authorize Observable to access the ${bold(repoName)} repository: ${link(authUrl)}`); + const spinner = this.effects.clack.spinner(); spinner.start("Waiting for repository to be authorized"); const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; - while (!authedRepo) { + while (!localRepo) { await new Promise((resolve) => setTimeout(resolve, 2000)); if (Date.now() > pollExpiration) { spinner.stop("Waiting for repository to be authorized timed out."); throw new CliError("Repository authorization failed"); } - authedRepo = await this.apiClient.getGitHubRepository(ownerName, repoName); - if (authedRepo) spinner.stop("Repository authorized."); + localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); + if (localRepo) spinner.stop("Repository authorized."); } } + const response = await this.apiClient.postProjectEnvironment(deployTarget.project.id, { source: { - provider: authedRepo.provider, - provider_id: authedRepo.provider_id, - url: authedRepo.url, + provider: localRepo.provider, + provider_id: localRepo.provider_id, + url: localRepo.url, branch } }); - return !!response; + + if (!response) throw new CliError("Setting source repository for continuous deployment failed"); + + // Configured repo is OK; proceed + return; } private async startNewDeploy(): Promise { @@ -514,8 +560,7 @@ class Deployer { continuousDeployment = enable; } - // Disables continuous deployment if there’s no env/source & we can’t link GitHub - if (continuousDeployment) await this.maybeLinkGitHub(deployTarget); + if (continuousDeployment) await this.validateGitHubLink(deployTarget); const newDeployConfig = { projectId: deployTarget.project.id, diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index 2f620b4ab..a539fae61 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -126,11 +126,20 @@ export class ObservableApiClient { return await this._fetch(url, {method: "GET"}); } - async getGitHubRepository(ownerName, repoName): Promise { - const url = new URL(`/cli/github/repository?owner=${ownerName}&repo=${repoName}`, this._apiOrigin); + async getGitHubRepository( + props: {ownerName: string; repoName: string} | {providerId: string} + ): Promise { + let url: URL; + if ("providerId" in props) { + url = new URL(`/cli/github/repository?provider_id=${props.providerId}`, this._apiOrigin); + } else { + url = new URL(`/cli/github/repository?owner=${props.ownerName}&repo=${props.repoName}`, this._apiOrigin); + } try { return await this._fetch(url, {method: "GET"}); } catch (err) { + // TODO: err.details.errors may be [{code: "NO_GITHUB_TOKEN"}] or [{code: "NO_REPO_ACCESS"}], + // which could be handled separately return null; } } From 5830afc023a19ef9ab534fe15c139b9bf3026762 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Thu, 31 Oct 2024 22:57:20 -0700 Subject: [PATCH 19/44] support SSH github remotes --- src/deploy.ts | 61 ++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 18 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 66c8f17ea..d53977b1d 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -49,6 +49,37 @@ function settingsUrl(deployTarget: DeployTargetInfo) { return `${OBSERVABLE_UI_ORIGIN}projects/@${deployTarget.workspace.login}/${deployTarget.project.slug}`; } +/** + * Returns the ownerName and repoName of the first GitHub remote (HTTPS or SSH) + * on the current repository, or null. + */ +async function getGitHubRemote() { + const remotes = (await promisify(exec)("git remote -v")).stdout + .split("\n") + .filter((d) => d) + .map((d) => { + const [, url] = d.split(/\s/g); + if (url.startsWith("https://github.com/")) { + // HTTPS: https://github.com/observablehq/framework.git + const [ownerName, repoName] = new URL(url).pathname + .slice(1) + .replace(/\.git$/, "") + .split("/"); + return {ownerName, repoName}; + } else if (url.startsWith("git@github.com:")) { + // SSH: git@github.com:observablehq/framework.git + const [ownerName, repoName] = url + .replace(/^git@github.com:/, "") + .replace(/\.git$/, "") + .split("/"); + return {ownerName, repoName}; + } + }); + const remote = remotes.find((d) => d && d.ownerName && d.repoName); + if (!remote) throw new CliError("No GitHub remote found."); + return remote ?? null; +} + export interface DeployOptions { config: Config; deployConfigPath: string | undefined; @@ -246,28 +277,23 @@ class Deployer { if (deployTarget.create) { throw new Error("Incorrect deploy target state"); } - // We only support cloud builds from the root directory so this ignores this.deployOptions.config.root + if (!deployTarget.project.build_environment_id) { + // TODO: allow setting build environment from CLI + throw new CliError("No build environment configured."); + } + // We only support cloud builds from the root directory so this ignores + // this.deployOptions.config.root const isGit = existsSync(".git"); if (!isGit) throw new CliError("Not at root of a git repository."); - const remotes = (await promisify(exec)("git remote -v", {cwd: this.deployOptions.config.root})).stdout - .split("\n") - .filter((d) => d) - .map((d) => d.split(/\s/g)); - const gitHub = remotes.find(([, url]) => url.startsWith("https://github.com/")); - if (!gitHub) throw new CliError("No GitHub remote found."); - // TODO: validate "Your branch is up to date" & "nothing to commit, working tree clean" - - if (!deployTarget.project.build_environment_id) throw new CliError("No build environment configured."); - // TODO: allow setting build environment from CLI - - const [ownerName, repoName] = formatGitUrl(gitHub[1]).split("/"); - const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD", {cwd: this.deployOptions.config.root})) - .stdout; + const {ownerName, repoName} = await getGitHubRemote(); + const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout; let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); // If a source repository has already been configured, check that it’s - // accessible and matches the local repository and branch + // accessible and matches the local repository and branch. + // TODO: validate local/remote refs match, "Your branch is up to date", + // and "nothing to commit, working tree clean". if (deployTarget.project.source) { if (localRepo && deployTarget.project.source.provider_id !== localRepo.provider_id) { throw new CliError( @@ -283,7 +309,6 @@ class Deployer { )}` ); } - // TODO: validate local/remote refs match const remoteAuthedRepo = await this.apiClient.getGitHubRepository({ providerId: deployTarget.project.source.provider_id }); @@ -295,7 +320,7 @@ class Deployer { )}` ); } - + // Configured repo is OK; proceed return; } From 048fd884484665947bda3ebf6e8dbf06566d2c2b Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Thu, 31 Oct 2024 23:01:55 -0700 Subject: [PATCH 20/44] move validateGitHubLink call to a better higher clearer more consolidated spot --- src/deploy.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index d53977b1d..7238d06f2 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -370,6 +370,7 @@ class Deployer { const {deployConfig, deployTarget} = await this.getDeployTarget(await this.getUpdatedDeployConfig()); let deployId: string | null; if (deployConfig.continuousDeployment) { + await this.validateGitHubLink(deployTarget); deployId = await this.cloudBuild(deployTarget); } else { const buildFilePaths = await this.getBuildFilePaths(); @@ -585,8 +586,6 @@ class Deployer { continuousDeployment = enable; } - if (continuousDeployment) await this.validateGitHubLink(deployTarget); - const newDeployConfig = { projectId: deployTarget.project.id, projectSlug: deployTarget.project.slug, From 7c719ae873e53137ca022f3e5ec120777ebb26f4 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Thu, 31 Oct 2024 23:03:40 -0700 Subject: [PATCH 21/44] tweak cd prompt to clarify that you need a github repo --- src/deploy.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/deploy.ts b/src/deploy.ts index 7238d06f2..3ddcf04e6 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -575,7 +575,7 @@ class Deployer { const enable = await this.effects.clack.confirm({ message: wrapAnsi( `Do you want to enable continuous deployment? ${faint( - "This builds in the cloud and redeploys whenever you push to this repository." + "Given a GitHub repository, this builds in the cloud and redeploys whenever you push to the current branch." )}`, this.effects.outputColumns ), From 8407f88d41092467ad73246379fad041e1473266 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 01:41:54 -0700 Subject: [PATCH 22/44] minimize diff --- src/deploy.ts | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 3ddcf04e6..46bdeee49 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -542,11 +542,7 @@ class Deployer { workspaceId: deployTarget.workspace.id, accessLevel: deployTarget.accessLevel }); - deployTarget = { - create: false, - workspace: deployTarget.workspace, - project - }; + deployTarget = {create: false, workspace: deployTarget.workspace, project}; } catch (error) { if (isApiError(error) && error.details.errors.some((e) => e.code === "TOO_MANY_PROJECTS")) { this.effects.clack.log.error( @@ -961,12 +957,7 @@ export async function promptDeployTarget( if (effects.clack.isCancel(chosenProject)) { throw new CliError("User canceled deploy.", {print: false, exitCode: 0}); } else if (chosenProject !== null) { - // TODO(toph): initial env config - return { - create: false, - workspace, - project: existingProjects.find((p) => p.slug === chosenProject)! - }; + return {create: false, workspace, project: existingProjects.find((p) => p.slug === chosenProject)!}; } } else { const confirmChoice = await effects.clack.confirm({ From 476ebeccca50d86adc65ae3ce398f592f6610626 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 01:42:57 -0700 Subject: [PATCH 23/44] minimize diff, again --- src/deploy.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 46bdeee49..f3ac98458 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -129,11 +129,7 @@ const defaultEffects: DeployEffects = { type DeployTargetInfo = | {create: true; workspace: {id: string; login: string}; projectSlug: string; title: string; accessLevel: string} - | { - create: false; - workspace: {id: string; login: string}; - project: GetProjectResponse; - }; + | {create: false; workspace: {id: string; login: string}; project: GetProjectResponse}; /** Deploy a project to Observable */ export async function deploy(deployOptions: DeployOptions, effects = defaultEffects): Promise { From bc60124ca2095b8a63397bcd515fb985fecb8570 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 18:37:24 -0700 Subject: [PATCH 24/44] first tests of validateGitHubLink --- src/deploy.ts | 13 +++-- test/deploy-test.ts | 109 ++++++++++++++++++++++++++++++------ test/mocks/observableApi.ts | 6 +- 3 files changed, 102 insertions(+), 26 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index f3ac98458..7f28f7bb8 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -290,15 +290,19 @@ class Deployer { // accessible and matches the local repository and branch. // TODO: validate local/remote refs match, "Your branch is up to date", // and "nothing to commit, working tree clean". - if (deployTarget.project.source) { - if (localRepo && deployTarget.project.source.provider_id !== localRepo.provider_id) { + const {source} = deployTarget.project; + if (source) { + if (localRepo && source.provider_id !== localRepo.provider_id) { throw new CliError( `Configured repository does not match local repository; check build settings on ${link( `${settingsUrl(deployTarget)}/settings` )}` ); } - if (localRepo && deployTarget.project.source.branch !== branch) { + if (localRepo && source.branch && source.branch !== branch) { + // TODO: If source.branch is empty, it'll use the default repository + // branch (usually main or master), which we don't know from our current + // getGitHubRepository response, and thus can't check here. throw new CliError( `Configured branch does not match local branch; check build settings on ${link( `${settingsUrl(deployTarget)}/settings` @@ -306,10 +310,9 @@ class Deployer { ); } const remoteAuthedRepo = await this.apiClient.getGitHubRepository({ - providerId: deployTarget.project.source.provider_id + providerId: source.provider_id }); if (!remoteAuthedRepo) { - console.log(deployTarget.project.source.provider_id, remoteAuthedRepo); throw new CliError( `Cannot access configured repository; check build settings on ${link( `${settingsUrl(deployTarget)}/settings` diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 76e95c31a..ace80d718 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -1,7 +1,11 @@ import assert, {fail} from "node:assert"; +import {exec} from "node:child_process"; import type {Stats} from "node:fs"; -import {stat} from "node:fs/promises"; +import {mkdtemp, rm, stat} from "node:fs/promises"; +import {tmpdir} from "node:os"; +import {join} from "node:path"; import {Readable, Writable} from "node:stream"; +import {promisify} from "node:util"; import type {BuildManifest} from "../src/build.js"; import {normalizeConfig, setCurrentDate} from "../src/config.js"; import type {DeployEffects, DeployOptions} from "../src/deploy.js"; @@ -115,7 +119,9 @@ class MockDeployEffects extends MockAuthEffects implements DeployEffects { async getDeployConfig(sourceRoot: string, deployConfigPath?: string): Promise { const key = this.getDeployConfigKey(sourceRoot, deployConfigPath); return ( - this.deployConfigs[key] ?? this.defaultDeployConfig ?? {projectId: null, projectSlug: null, workspaceLogin: null} + this.deployConfigs[key] ?? + this.defaultDeployConfig ?? + ({projectId: null, projectSlug: null, workspaceLogin: null, continuousDeployment: null} satisfies DeployConfig) ); } @@ -194,11 +200,89 @@ const DEPLOY_CONFIG: DeployConfig & {projectId: string; projectSlug: string; wor continuousDeployment: false }; +function mockIsolatedDirectory({git}: {git: boolean}) { + let dir: string; + let cwd: string; + beforeEach(async () => { + cwd = process.cwd(); + dir = await mkdtemp(join(tmpdir(), "framework-test-")); + process.chdir(dir); + if (git) (await promisify(exec)("git init")).stdout; + }); + + afterEach(async () => { + process.chdir(cwd); + await rm(dir, {recursive: true}); + }); +} + describe("deploy", () => { before(() => setCurrentDate(new Date("2024-01-10T16:00:00"))); mockObservableApi(); mockJsDelivr(); + describe("in isolated directory with git repo", () => { + mockIsolatedDirectory({git: true}); + + it("fails continuous deployment if repo has no GitHub remote", async () => { + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetWorkspaceProjects({ + workspaceLogin: DEPLOY_CONFIG.workspaceLogin, + projects: [] + }) + .handlePostProject({projectId: DEPLOY_CONFIG.projectId}) + .start(); + const effects = new MockDeployEffects(); + effects.clack.inputs.push( + true, // No apps found. Do you want to create a new app? + "cloud-deployed-app", // What slug do you want to use? + "public", // Who is allowed to access your app? + true // Do you want to enable continuous deployment? + ); + + try { + await deploy(TEST_OPTIONS, effects); + assert.fail("expected error"); + } catch (error) { + CliError.assert(error, {message: "No GitHub remote found."}); + } + + effects.close(); + }); + }); + + describe("in isolated directory without git repo", () => { + mockIsolatedDirectory({git: false}); + + it("fails continuous deployment if not in a git repo", async () => { + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetWorkspaceProjects({ + workspaceLogin: DEPLOY_CONFIG.workspaceLogin, + projects: [] + }) + .handlePostProject({projectId: DEPLOY_CONFIG.projectId}) + .start(); + const effects = new MockDeployEffects(); + effects.clack.inputs.push( + true, // No apps found. Do you want to create a new app? + "cloud-deployed-app", // What slug do you want to use? + "public", // Who is allowed to access your app? + true // Do you want to enable continuous deployment? + ); + + try { + await deploy(TEST_OPTIONS, effects); + assert.fail("expected error"); + } catch (error) { + CliError.assert(error, {message: "Not at root of a git repository."}); + } + + effects.close(); + }); + }); + it("makes expected API calls for an existing project", async () => { const deployId = "deploy456"; getCurrentObservableApi() @@ -334,6 +418,7 @@ describe("deploy", () => { effects.clack.inputs.push( DEPLOY_CONFIG.projectSlug, // which project do you want to use? true, // Do you want to continue? (and overwrite the project) + false, // Do you want to enable continuous deployment? "change project title" // "what changed?" ); await deploy(TEST_OPTIONS, effects); @@ -887,10 +972,7 @@ describe("deploy", () => { force: null, config: {...TEST_OPTIONS.config, output: "test/output/does-not-exist"} } satisfies DeployOptions; - getCurrentObservableApi() - .handleGetCurrentUser() - .handleGetProject(DEPLOY_CONFIG) - .start(); + getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-09"), @@ -905,10 +987,7 @@ describe("deploy", () => { ...TEST_OPTIONS, force: null } satisfies DeployOptions; - getCurrentObservableApi() - .handleGetCurrentUser() - .handleGetProject(DEPLOY_CONFIG) - .start(); + getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-09"), @@ -926,10 +1005,7 @@ describe("deploy", () => { ...TEST_OPTIONS, force: null } satisfies DeployOptions; - getCurrentObservableApi() - .handleGetCurrentUser() - .handleGetProject(DEPLOY_CONFIG) - .start(); + getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-11"), @@ -947,10 +1023,7 @@ describe("deploy", () => { ...TEST_OPTIONS, force: "build" } satisfies DeployOptions; - getCurrentObservableApi() - .handleGetCurrentUser() - .handleGetProject(DEPLOY_CONFIG) - .start(); + getCurrentObservableApi().handleGetCurrentUser().handleGetProject(DEPLOY_CONFIG).start(); const effects = new MockDeployEffects({ deployConfig: DEPLOY_CONFIG, fixedInputStatTime: new Date("2024-03-09"), diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts index 42071bc0a..07ea3713c 100644 --- a/test/mocks/observableApi.ts +++ b/test/mocks/observableApi.ts @@ -169,7 +169,7 @@ class ObservableApiMock { owner: {id: "workspace-id", login: "workspace-login"}, latestCreatedDeployId: null, automatic_builds_enabled: null, - build_environment_id: null, + build_environment_id: "abc123", source: null } satisfies GetProjectResponse) : emptyErrorBody; @@ -209,8 +209,8 @@ class ObservableApiMock { owner, creator, latestCreatedDeployId: null, - automatic_builds_enabled: null, - build_environment_id: null, + automatic_builds_enabled: true, + build_environment_id: "abc123", source: null } satisfies GetProjectResponse) : emptyErrorBody; From 14ce19bfb8549769dc7213701a48b78ee60daff4 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 19:40:19 -0700 Subject: [PATCH 25/44] whoohoo end-to-end test of kicking off cloud build --- src/deploy.ts | 4 --- test/deploy-test.ts | 39 ++++++++++++++++++++-- test/mocks/observableApi.ts | 66 +++++++++++++++++++++++++++++++++++-- 3 files changed, 100 insertions(+), 9 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 7f28f7bb8..dbeda2a92 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -38,10 +38,6 @@ const BUILD_AGE_WARNING_MS = 1000 * 60 * 5; const OBSERVABLE_UI_ORIGIN = getObservableUiOrigin(); -export function formatGitUrl(url: string) { - return new URL(url).pathname.slice(1).replace(/\.git$/, ""); -} - function settingsUrl(deployTarget: DeployTargetInfo) { if (deployTarget.create) { throw new Error("Incorrect deploy target state"); diff --git a/test/deploy-test.ts b/test/deploy-test.ts index ace80d718..371a1350f 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -231,12 +231,12 @@ describe("deploy", () => { workspaceLogin: DEPLOY_CONFIG.workspaceLogin, projects: [] }) - .handlePostProject({projectId: DEPLOY_CONFIG.projectId}) + .handlePostProject({projectId: DEPLOY_CONFIG.projectId, slug: DEPLOY_CONFIG.projectSlug}) .start(); const effects = new MockDeployEffects(); effects.clack.inputs.push( true, // No apps found. Do you want to create a new app? - "cloud-deployed-app", // What slug do you want to use? + DEPLOY_CONFIG.projectSlug, // What slug do you want to use? "public", // Who is allowed to access your app? true // Do you want to enable continuous deployment? ); @@ -250,6 +250,39 @@ describe("deploy", () => { effects.close(); }); + + it("starts cloud build when continuous deployment is enabled and repo is valid", async () => { + const deployId = "deploy123"; + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetWorkspaceProjects({ + workspaceLogin: DEPLOY_CONFIG.workspaceLogin, + projects: [] + }) + .handlePostProject({projectId: DEPLOY_CONFIG.projectId, slug: DEPLOY_CONFIG.projectSlug}) + .handleGetRepository() + .handlePostProjectEnvironment() + .handlePostProjectBuild() + .handleGetProject({...DEPLOY_CONFIG, latestCreatedDeployId: deployId}) + .handleGetDeploy({deployId, deployStatus: "uploaded"}) + .start(); + const effects = new MockDeployEffects(); + effects.clack.inputs.push( + true, // No apps found. Do you want to create a new app? + DEPLOY_CONFIG.projectSlug, // What slug do you want to use? + "public", // Who is allowed to access your app? + true // Do you want to enable continuous deployment? + ); + + await promisify(exec)( + "touch readme.md; git add .; git commit -m 'initial'; git remote add origin git@github.com:observablehq/test.git" + ); + + await deploy(TEST_OPTIONS, effects); + + effects.close(); + }); + }); describe("in isolated directory without git repo", () => { @@ -267,7 +300,7 @@ describe("deploy", () => { const effects = new MockDeployEffects(); effects.clack.inputs.push( true, // No apps found. Do you want to create a new app? - "cloud-deployed-app", // What slug do you want to use? + DEPLOY_CONFIG.projectSlug, // What slug do you want to use? "public", // Who is allowed to access your app? true // Do you want to enable continuous deployment? ); diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts index 07ea3713c..23bca3147 100644 --- a/test/mocks/observableApi.ts +++ b/test/mocks/observableApi.ts @@ -149,6 +149,7 @@ class ObservableApiMock { projectId = "project123", title = "Build test case", accessLevel = "private", + latestCreatedDeployId = null, status = 200 }: { workspaceLogin: string; @@ -157,6 +158,7 @@ class ObservableApiMock { title?: string; accessLevel?: string; status?: number; + latestCreatedDeployId?: null | string; }): ObservableApiMock { const response = status === 200 @@ -167,8 +169,8 @@ class ObservableApiMock { title, creator: {id: "user-id", login: "user-login"}, owner: {id: "workspace-id", login: "workspace-login"}, - latestCreatedDeployId: null, - automatic_builds_enabled: null, + latestCreatedDeployId, + automatic_builds_enabled: true, build_environment_id: "abc123", source: null } satisfies GetProjectResponse) @@ -432,6 +434,66 @@ class ObservableApiMock { ); return this; } + + handleGetRepository({status = 200}: {status?: number} = {}) { + const response = + status === 200 + ? JSON.stringify({ + provider: "github", + provider_id: "123:456", + url: "https://github.com/observablehq/test.git", + default_branch: "main", + name: "test", + linked_projects: [] + }) + : emptyErrorBody; + const headers = authorizationHeader(status !== 401); + this._handlers.push((pool) => + pool + .intercept({path: "/cli/github/repository?owner=observablehq&repo=test", headers: headersMatcher(headers)}) + .reply(status, response, {headers: {"content-type": "application/json"}}) + ); + return this; + } + + handlePostProjectEnvironment({status = 200}: {status?: number} = {}) { + const response = + status === 200 + ? JSON.stringify({ + automatic_builds_enabled: true, + build_environment_id: "abc123", + source: { + provider: "github", + provider_id: "123:456", + url: "https://github.com/observablehq/test.git", + branch: "main" + } + }) + : emptyErrorBody; + const headers = authorizationHeader(status !== 401); + this._handlers.push((pool) => + pool + .intercept({path: "/cli/project/project123/environment", method: "POST", headers: headersMatcher(headers)}) + .reply(status, response, {headers: {"content-type": "application/json"}}) + ); + return this; + } + + handlePostProjectBuild({status = 200}: {status?: number} = {}) { + const response = + status === 200 + ? JSON.stringify({ + id: "abc123" + }) + : emptyErrorBody; + const headers = authorizationHeader(status !== 401); + this._handlers.push((pool) => + pool + .intercept({path: "/cli/project/project123/build", method: "POST", headers: headersMatcher(headers)}) + .reply(status, response, {headers: {"content-type": "application/json"}}) + ); + return this; + } } function authorizationHeader(valid: boolean) { From 27128bbc212ac30fa7d7ffc75f20d7d1e3f051de Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 20:34:26 -0700 Subject: [PATCH 26/44] ALL TESTS PASSING incl coverage threshold, thanks to testing cloud build of preconfigured project --- src/deploy.ts | 2 +- test/deploy-test.ts | 41 +++++++++++++++++++++++++++++++++++++ test/mocks/observableApi.ts | 31 ++++++++++++++++++++-------- 3 files changed, 65 insertions(+), 9 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index dbeda2a92..b4f6cb666 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -279,7 +279,7 @@ class Deployer { if (!isGit) throw new CliError("Not at root of a git repository."); const {ownerName, repoName} = await getGitHubRemote(); - const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout; + const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout.trim(); let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); // If a source repository has already been configured, check that it’s diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 371a1350f..4c4791e7c 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -283,6 +283,47 @@ describe("deploy", () => { effects.close(); }); + it("starts cloud build when continuous deployment is enabled for existing project with existing source", async () => { + const deployId = "deploy123"; + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetProject({ + ...DEPLOY_CONFIG, + source: { + provider: "github", + provider_id: "123:456", + url: "https://github.com/observablehq/test.git", + branch: "main" + }, + latestCreatedDeployId: null + }) + .handleGetRepository({useProviderId: true}) + .handlePostProjectBuild() + .handleGetProject({ + ...DEPLOY_CONFIG, + source: { + provider: "github", + provider_id: "123:456", + url: "https://github.com/observablehq/test.git", + branch: "main" + }, + latestCreatedDeployId: deployId + }) + .handleGetDeploy({deployId, deployStatus: "uploaded"}) + .start(); + const effects = new MockDeployEffects({deployConfig: {...DEPLOY_CONFIG, continuousDeployment: true}}); + effects.clack.inputs.push( + "bi" // Which app do you want to use? + ); + + await promisify(exec)( + "touch readme.md; git add .; git commit -m 'initial'; git remote add origin https://github.com/observablehq/test.git" + ); + + await deploy(TEST_OPTIONS, effects); + + effects.close(); + }); }); describe("in isolated directory without git repo", () => { diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts index 23bca3147..3e08541d2 100644 --- a/test/mocks/observableApi.ts +++ b/test/mocks/observableApi.ts @@ -150,6 +150,7 @@ class ObservableApiMock { title = "Build test case", accessLevel = "private", latestCreatedDeployId = null, + source = null, status = 200 }: { workspaceLogin: string; @@ -158,6 +159,7 @@ class ObservableApiMock { title?: string; accessLevel?: string; status?: number; + source?: GetProjectResponse["source"]; latestCreatedDeployId?: null | string; }): ObservableApiMock { const response = @@ -168,11 +170,11 @@ class ObservableApiMock { slug: projectSlug, title, creator: {id: "user-id", login: "user-login"}, - owner: {id: "workspace-id", login: "workspace-login"}, + owner: {id: "workspace-id", login: workspaceLogin}, latestCreatedDeployId, automatic_builds_enabled: true, build_environment_id: "abc123", - source: null + source } satisfies GetProjectResponse) : emptyErrorBody; const headers = authorizationHeader(status !== 401 && status !== 403); @@ -435,7 +437,7 @@ class ObservableApiMock { return this; } - handleGetRepository({status = 200}: {status?: number} = {}) { + handleGetRepository({status = 200, useProviderId = false}: {status?: number; useProviderId?: boolean} = {}) { const response = status === 200 ? JSON.stringify({ @@ -448,11 +450,24 @@ class ObservableApiMock { }) : emptyErrorBody; const headers = authorizationHeader(status !== 401); - this._handlers.push((pool) => - pool - .intercept({path: "/cli/github/repository?owner=observablehq&repo=test", headers: headersMatcher(headers)}) - .reply(status, response, {headers: {"content-type": "application/json"}}) - ); + if (useProviderId) { + // version that accepts provider_id + this._handlers.push((pool) => + pool + .intercept({ + path: `/cli/github/repository?provider_id=${encodeURIComponent("123:456")}`, + headers: headersMatcher(headers) + }) + .reply(status, response, {headers: {"content-type": "application/json"}}) + ); + } else { + // version that accepts owner & repo + this._handlers.push((pool) => + pool + .intercept({path: "/cli/github/repository?owner=observablehq&repo=test", headers: headersMatcher(headers)}) + .reply(status, response, {headers: {"content-type": "application/json"}}) + ); + } return this; } From 82f7b0d3094a689704a7bcbe3f86d9e4732cca69 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 20:55:08 -0700 Subject: [PATCH 27/44] move mockIsolatedDirectory to own file --- test/deploy-test.ts | 21 ++------------------- test/mocks/directory.ts | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 19 deletions(-) create mode 100644 test/mocks/directory.ts diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 4c4791e7c..49de47436 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -1,9 +1,7 @@ import assert, {fail} from "node:assert"; import {exec} from "node:child_process"; import type {Stats} from "node:fs"; -import {mkdtemp, rm, stat} from "node:fs/promises"; -import {tmpdir} from "node:os"; -import {join} from "node:path"; +import {stat} from "node:fs/promises"; import {Readable, Writable} from "node:stream"; import {promisify} from "node:util"; import type {BuildManifest} from "../src/build.js"; @@ -20,6 +18,7 @@ import {stripColor} from "../src/tty.js"; import {MockAuthEffects} from "./mocks/authEffects.js"; import {TestClackEffects} from "./mocks/clack.js"; import {MockConfigEffects} from "./mocks/configEffects.js"; +import {mockIsolatedDirectory} from "./mocks/directory.js"; import {mockJsDelivr} from "./mocks/jsdelivr.js"; import {MockLogger} from "./mocks/logger.js"; import { @@ -200,22 +199,6 @@ const DEPLOY_CONFIG: DeployConfig & {projectId: string; projectSlug: string; wor continuousDeployment: false }; -function mockIsolatedDirectory({git}: {git: boolean}) { - let dir: string; - let cwd: string; - beforeEach(async () => { - cwd = process.cwd(); - dir = await mkdtemp(join(tmpdir(), "framework-test-")); - process.chdir(dir); - if (git) (await promisify(exec)("git init")).stdout; - }); - - afterEach(async () => { - process.chdir(cwd); - await rm(dir, {recursive: true}); - }); -} - describe("deploy", () => { before(() => setCurrentDate(new Date("2024-01-10T16:00:00"))); mockObservableApi(); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts new file mode 100644 index 000000000..817a0fa3b --- /dev/null +++ b/test/mocks/directory.ts @@ -0,0 +1,21 @@ +import { exec } from "child_process"; +import { mkdtemp, rm } from "fs/promises"; +import { tmpdir } from "os"; +import { join } from "path/posix"; +import { promisify } from "util"; + +export function mockIsolatedDirectory({ git }: { git: boolean; }) { + let dir: string; + let cwd: string; + beforeEach(async () => { + cwd = process.cwd(); + dir = await mkdtemp(join(tmpdir(), "framework-test-")); + process.chdir(dir); + if (git) (await promisify(exec)("git init")).stdout; + }); + + afterEach(async () => { + process.chdir(cwd); + await rm(dir, { recursive: true }); + }); +} From e511cc7f21985b9ffb3c3c7f501c24aa65321238 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:05:44 -0700 Subject: [PATCH 28/44] testing if git is installed on ubuntu --- test/mocks/directory.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index 817a0fa3b..1353d26e6 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -11,7 +11,10 @@ export function mockIsolatedDirectory({ git }: { git: boolean; }) { cwd = process.cwd(); dir = await mkdtemp(join(tmpdir(), "framework-test-")); process.chdir(dir); - if (git) (await promisify(exec)("git init")).stdout; + if (git) { + const {stdout, stderr} = (await promisify(exec)("git init")); + console.log({stdout, stderr}); + }; }); afterEach(async () => { From bb9c311319aa74766231ea873cfd579c47d17139 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:12:22 -0700 Subject: [PATCH 29/44] testing deterministic default branch name --- test/mocks/directory.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index 1353d26e6..847a8c616 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -12,8 +12,10 @@ export function mockIsolatedDirectory({ git }: { git: boolean; }) { dir = await mkdtemp(join(tmpdir(), "framework-test-")); process.chdir(dir); if (git) { - const {stdout, stderr} = (await promisify(exec)("git init")); - console.log({stdout, stderr}); + const a = (await promisify(exec)("git config --global init.defaultBranch main")) + console.log(a.stdout, a.stderr); + const b = (await promisify(exec)("git init")); + console.log(b.stdout, b.stderr); }; }); From ff60b8075aaae102c25383fdf31db83c3eacd05c Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:18:33 -0700 Subject: [PATCH 30/44] more debugging... --- test/deploy-test.ts | 3 ++- test/mocks/directory.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 49de47436..8352d0078 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -257,9 +257,10 @@ describe("deploy", () => { true // Do you want to enable continuous deployment? ); - await promisify(exec)( + const {stdout, stderr} = await promisify(exec)( "touch readme.md; git add .; git commit -m 'initial'; git remote add origin git@github.com:observablehq/test.git" ); + console.log({stdout, stderr}); await deploy(TEST_OPTIONS, effects); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index 847a8c616..e75622d21 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -12,6 +12,7 @@ export function mockIsolatedDirectory({ git }: { git: boolean; }) { dir = await mkdtemp(join(tmpdir(), "framework-test-")); process.chdir(dir); if (git) { + console.log("logging stdout, stderr"); const a = (await promisify(exec)("git config --global init.defaultBranch main")) console.log(a.stdout, a.stderr); const b = (await promisify(exec)("git init")); From f1cd74dff640a9041fae5b8842fdbc27e0923f3c Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:27:25 -0700 Subject: [PATCH 31/44] setting name and email config for git, seeing if that helps --- src/deploy.ts | 4 +++- test/deploy-test.ts | 2 +- test/mocks/directory.ts | 22 ++++++++++++---------- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index b4f6cb666..2cb26cb61 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -279,7 +279,9 @@ class Deployer { if (!isGit) throw new CliError("Not at root of a git repository."); const {ownerName, repoName} = await getGitHubRemote(); - const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout.trim(); + const a = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")); + console.log("validateGitHubLink", {stdout: a.stdout, stderr: a.stderr}); + const branch = a.stdout.trim(); let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); // If a source repository has already been configured, check that it’s diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 8352d0078..6259f90d7 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -260,7 +260,7 @@ describe("deploy", () => { const {stdout, stderr} = await promisify(exec)( "touch readme.md; git add .; git commit -m 'initial'; git remote add origin git@github.com:observablehq/test.git" ); - console.log({stdout, stderr}); + console.log("starts cloud build test", {stdout, stderr}); await deploy(TEST_OPTIONS, effects); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index e75622d21..dbb6cb0e4 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -1,10 +1,10 @@ -import { exec } from "child_process"; -import { mkdtemp, rm } from "fs/promises"; -import { tmpdir } from "os"; -import { join } from "path/posix"; -import { promisify } from "util"; +import {exec} from "child_process"; +import {mkdtemp, rm} from "fs/promises"; +import {tmpdir} from "os"; +import {join} from "path/posix"; +import {promisify} from "util"; -export function mockIsolatedDirectory({ git }: { git: boolean; }) { +export function mockIsolatedDirectory({git}: {git: boolean}) { let dir: string; let cwd: string; beforeEach(async () => { @@ -13,15 +13,17 @@ export function mockIsolatedDirectory({ git }: { git: boolean; }) { process.chdir(dir); if (git) { console.log("logging stdout, stderr"); - const a = (await promisify(exec)("git config --global init.defaultBranch main")) + const a = await promisify(exec)( + "git config --global user.email \"you@example.com\"; git config --global user.name \"Your Name\"; git config --global init.defaultBranch main" + ); console.log(a.stdout, a.stderr); - const b = (await promisify(exec)("git init")); + const b = await promisify(exec)("git init"); console.log(b.stdout, b.stderr); - }; + } }); afterEach(async () => { process.chdir(cwd); - await rm(dir, { recursive: true }); + await rm(dir, {recursive: true}); }); } From e188a3940fdb2d7134f71b45184c0267d4c8c96b Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:35:45 -0700 Subject: [PATCH 32/44] fix command separator from ; to && for windows; prettier for ubuntu --- test/deploy-test.ts | 2 +- test/mocks/directory.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 6259f90d7..d24f5bc54 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -258,7 +258,7 @@ describe("deploy", () => { ); const {stdout, stderr} = await promisify(exec)( - "touch readme.md; git add .; git commit -m 'initial'; git remote add origin git@github.com:observablehq/test.git" + "touch readme.md && git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" ); console.log("starts cloud build test", {stdout, stderr}); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index dbb6cb0e4..2698d7e25 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -14,7 +14,7 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { if (git) { console.log("logging stdout, stderr"); const a = await promisify(exec)( - "git config --global user.email \"you@example.com\"; git config --global user.name \"Your Name\"; git config --global init.defaultBranch main" + 'git config --global user.email "observable@example.com" && git config --global user.name "Observable User" && git config --global init.defaultBranch main' ); console.log(a.stdout, a.stderr); const b = await promisify(exec)("git init"); From fae07e14cf719aaa877c9ea04935c0aabd034c50 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:48:10 -0700 Subject: [PATCH 33/44] use fs instead of touch for cross-platform (windows) compatibility; fix prettttier thing i missed --- src/deploy.ts | 2 +- test/deploy-test.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index 2cb26cb61..73d414a27 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -279,7 +279,7 @@ class Deployer { if (!isGit) throw new CliError("Not at root of a git repository."); const {ownerName, repoName} = await getGitHubRemote(); - const a = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")); + const a = await promisify(exec)("git rev-parse --abbrev-ref HEAD"); console.log("validateGitHubLink", {stdout: a.stdout, stderr: a.stderr}); const branch = a.stdout.trim(); let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); diff --git a/test/deploy-test.ts b/test/deploy-test.ts index d24f5bc54..4a5722ce9 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -1,7 +1,7 @@ import assert, {fail} from "node:assert"; import {exec} from "node:child_process"; import type {Stats} from "node:fs"; -import {stat} from "node:fs/promises"; +import {open, stat} from "node:fs/promises"; import {Readable, Writable} from "node:stream"; import {promisify} from "node:util"; import type {BuildManifest} from "../src/build.js"; @@ -257,8 +257,11 @@ describe("deploy", () => { true // Do you want to enable continuous deployment? ); + await ( + await open("readme.md", "a") + ).close; const {stdout, stderr} = await promisify(exec)( - "touch readme.md && git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" + "git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" ); console.log("starts cloud build test", {stdout, stderr}); From 41f96821b0f92b1b65d29252dadc2fd269552738 Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 21:52:07 -0700 Subject: [PATCH 34/44] force: true on rm dir for windows complaining ENOTEMPTY --- test/mocks/directory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index 2698d7e25..a1e731f07 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -24,6 +24,6 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { afterEach(async () => { process.chdir(cwd); - await rm(dir, {recursive: true}); + await rm(dir, {recursive: true, force: true}); }); } From dd50d70f94ef70c43fabbb51ccd7fc5166defe64 Mon Sep 17 00:00:00 2001 From: Observable User Date: Fri, 1 Nov 2024 21:59:36 -0700 Subject: [PATCH 35/44] adopting rimraf in lieu of node fs rm for what i hope is better windows compatibility --- test/mocks/directory.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index a1e731f07..f90750e88 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -3,6 +3,7 @@ import {mkdtemp, rm} from "fs/promises"; import {tmpdir} from "os"; import {join} from "path/posix"; import {promisify} from "util"; +import {rimraf} from "rimraf"; export function mockIsolatedDirectory({git}: {git: boolean}) { let dir: string; @@ -24,6 +25,7 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { afterEach(async () => { process.chdir(cwd); - await rm(dir, {recursive: true, force: true}); + await rimraf(dir); + // await rm(dir, {recursive: true, force: true}); }); } From 3f0bbb8981284a2b3872022951c5a121e206fee0 Mon Sep 17 00:00:00 2001 From: Observable User Date: Fri, 1 Nov 2024 22:05:21 -0700 Subject: [PATCH 36/44] oops i wasnt actually closing the file i made. good call, windows. rolling back other forcing things to see if i still need them --- test/deploy-test.ts | 4 +--- test/mocks/directory.ts | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 4a5722ce9..4ff60f7bc 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -257,9 +257,7 @@ describe("deploy", () => { true // Do you want to enable continuous deployment? ); - await ( - await open("readme.md", "a") - ).close; + await (await open("readme.md", "a")).close(); const {stdout, stderr} = await promisify(exec)( "git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" ); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index f90750e88..adb06582e 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -25,7 +25,7 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { afterEach(async () => { process.chdir(cwd); - await rimraf(dir); - // await rm(dir, {recursive: true, force: true}); + // await rimraf(dir); + await rm(dir, {recursive: true}); //, force: true }); } From fede9265b8ea730e9f0abe49e0499750d529a082 Mon Sep 17 00:00:00 2001 From: Observable User Date: Fri, 1 Nov 2024 22:09:55 -0700 Subject: [PATCH 37/44] ah i had another case of touch --- test/deploy-test.ts | 3 ++- test/mocks/directory.ts | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 4ff60f7bc..ad664b7ec 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -301,8 +301,9 @@ describe("deploy", () => { "bi" // Which app do you want to use? ); + await (await open("readme.md", "a")).close(); await promisify(exec)( - "touch readme.md; git add .; git commit -m 'initial'; git remote add origin https://github.com/observablehq/test.git" + "git add . && git commit -m 'initial' && git remote add origin https://github.com/observablehq/test.git" ); await deploy(TEST_OPTIONS, effects); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index adb06582e..e79659312 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -3,7 +3,7 @@ import {mkdtemp, rm} from "fs/promises"; import {tmpdir} from "os"; import {join} from "path/posix"; import {promisify} from "util"; -import {rimraf} from "rimraf"; +// import {rimraf} from "rimraf"; export function mockIsolatedDirectory({git}: {git: boolean}) { let dir: string; From 172bf50cdd38b62b23020caf0f34b4e024c67756 Mon Sep 17 00:00:00 2001 From: Observable User Date: Fri, 1 Nov 2024 22:32:01 -0700 Subject: [PATCH 38/44] new test for when repo doesnt match --- test/deploy-test.ts | 44 +++++++++++++++++++++++++++++++------ test/mocks/observableApi.ts | 19 +++++++++++----- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/test/deploy-test.ts b/test/deploy-test.ts index ad664b7ec..1f0f067ed 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -14,7 +14,7 @@ import type {ObservableApiClientOptions, PostDeployUploadedRequest} from "../src import type {GetCurrentUserResponse} from "../src/observableApiClient.js"; import {ObservableApiClient} from "../src/observableApiClient.js"; import type {DeployConfig} from "../src/observableApiConfig.js"; -import {stripColor} from "../src/tty.js"; +import {link, stripColor} from "../src/tty.js"; import {MockAuthEffects} from "./mocks/authEffects.js"; import {TestClackEffects} from "./mocks/clack.js"; import {MockConfigEffects} from "./mocks/configEffects.js"; @@ -207,7 +207,7 @@ describe("deploy", () => { describe("in isolated directory with git repo", () => { mockIsolatedDirectory({git: true}); - it("fails continuous deployment if repo has no GitHub remote", async () => { + it("fails cloud build if repo has no GitHub remote", async () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetWorkspaceProjects({ @@ -234,7 +234,39 @@ describe("deploy", () => { effects.close(); }); - it("starts cloud build when continuous deployment is enabled and repo is valid", async () => { + it("fails cloud build if repo doesn’t match", async () => { + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetProject({ + ...DEPLOY_CONFIG, + source: { + provider: "github", + provider_id: "123:456", + url: "https://github.com/observablehq/test.git", + branch: "main" + }, + latestCreatedDeployId: null + }) + .handleGetRepository({ownerName: "observablehq", repoName: "wrongrepo", provider_id: "000:001"}) + .start(); + const effects = new MockDeployEffects({deployConfig: {...DEPLOY_CONFIG, continuousDeployment: true}}); + + await (await open("readme.md", "a")).close(); + await promisify(exec)( + "git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/wrongrepo.git" + ); + + try { + await deploy(TEST_OPTIONS, effects); + assert.fail("expected error"); + } catch (error) { + CliError.assert(error, {message: `Configured repository does not match local repository; check build settings on ${link(`https://observablehq.com/projects/@${DEPLOY_CONFIG.workspaceLogin}/${DEPLOY_CONFIG.projectSlug}/settings`)}`}); + } + + effects.close(); + }); + + it("starts cloud build when continuous deployment is enabled for new project and repo is valid", async () => { const deployId = "deploy123"; getCurrentObservableApi() .handleGetCurrentUser() @@ -282,6 +314,7 @@ describe("deploy", () => { }, latestCreatedDeployId: null }) + .handleGetRepository({useProviderId: false}) .handleGetRepository({useProviderId: true}) .handlePostProjectBuild() .handleGetProject({ @@ -297,9 +330,6 @@ describe("deploy", () => { .handleGetDeploy({deployId, deployStatus: "uploaded"}) .start(); const effects = new MockDeployEffects({deployConfig: {...DEPLOY_CONFIG, continuousDeployment: true}}); - effects.clack.inputs.push( - "bi" // Which app do you want to use? - ); await (await open("readme.md", "a")).close(); await promisify(exec)( @@ -315,7 +345,7 @@ describe("deploy", () => { describe("in isolated directory without git repo", () => { mockIsolatedDirectory({git: false}); - it("fails continuous deployment if not in a git repo", async () => { + it("fails cloud build if not in a git repo", async () => { getCurrentObservableApi() .handleGetCurrentUser() .handleGetWorkspaceProjects({ diff --git a/test/mocks/observableApi.ts b/test/mocks/observableApi.ts index 3e08541d2..2ac1bb5e5 100644 --- a/test/mocks/observableApi.ts +++ b/test/mocks/observableApi.ts @@ -437,13 +437,19 @@ class ObservableApiMock { return this; } - handleGetRepository({status = 200, useProviderId = false}: {status?: number; useProviderId?: boolean} = {}) { + handleGetRepository({ + status = 200, + ownerName = "observablehq", + repoName = "test", + provider_id = "123:456", + useProviderId = false + }: {status?: number; ownerName?: string; repoName?: string; provider_id?: string; useProviderId?: boolean} = {}) { const response = status === 200 ? JSON.stringify({ provider: "github", - provider_id: "123:456", - url: "https://github.com/observablehq/test.git", + provider_id, + url: `https://github.com/${ownerName}/${repoName}.git`, default_branch: "main", name: "test", linked_projects: [] @@ -455,7 +461,7 @@ class ObservableApiMock { this._handlers.push((pool) => pool .intercept({ - path: `/cli/github/repository?provider_id=${encodeURIComponent("123:456")}`, + path: `/cli/github/repository?provider_id=${encodeURIComponent(provider_id)}`, headers: headersMatcher(headers) }) .reply(status, response, {headers: {"content-type": "application/json"}}) @@ -464,7 +470,10 @@ class ObservableApiMock { // version that accepts owner & repo this._handlers.push((pool) => pool - .intercept({path: "/cli/github/repository?owner=observablehq&repo=test", headers: headersMatcher(headers)}) + .intercept({ + path: `/cli/github/repository?owner=${ownerName}&repo=${repoName}`, + headers: headersMatcher(headers) + }) .reply(status, response, {headers: {"content-type": "application/json"}}) ); } From 87f707d940db05f5540bbeeb6e90b28089a41df8 Mon Sep 17 00:00:00 2001 From: Observable User Date: Fri, 1 Nov 2024 22:44:03 -0700 Subject: [PATCH 39/44] add test for polling for repo auth; deploy.ts test coverage is now back up to status quo ante; fix polling for auth to respect deployOptions.deployPollInterval, though one could argue its now too narrowly named --- src/deploy.ts | 5 ++++- test/deploy-test.ts | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/src/deploy.ts b/src/deploy.ts index 73d414a27..d135c1ca5 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -311,6 +311,8 @@ class Deployer { providerId: source.provider_id }); if (!remoteAuthedRepo) { + // TODO: This could poll for auth too, but is a distinct case because it + // means the repo was linked at one point and then something went wrong throw new CliError( `Cannot access configured repository; check build settings on ${link( `${settingsUrl(deployTarget)}/settings` @@ -336,9 +338,10 @@ class Deployer { const spinner = this.effects.clack.spinner(); spinner.start("Waiting for repository to be authorized"); + const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; while (!localRepo) { - await new Promise((resolve) => setTimeout(resolve, 2000)); + await new Promise((resolve) => setTimeout(resolve, pollInterval)); if (Date.now() > pollExpiration) { spinner.stop("Waiting for repository to be authorized timed out."); throw new CliError("Repository authorization failed"); diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 1f0f067ed..5a6c82b49 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -300,6 +300,41 @@ describe("deploy", () => { effects.close(); }); + it("starts cloud build when continuous deployment is enabled for new project and repo is manually auth’ed while CLI is polling", async () => { + const deployId = "deploy123"; + getCurrentObservableApi() + .handleGetCurrentUser() + .handleGetWorkspaceProjects({ + workspaceLogin: DEPLOY_CONFIG.workspaceLogin, + projects: [] + }) + .handlePostProject({projectId: DEPLOY_CONFIG.projectId, slug: DEPLOY_CONFIG.projectSlug}) + .handleGetRepository({status: 404}) + .handleGetRepository() + .handlePostProjectEnvironment() + .handlePostProjectBuild() + .handleGetProject({...DEPLOY_CONFIG, latestCreatedDeployId: deployId}) + .handleGetDeploy({deployId, deployStatus: "uploaded"}) + .start(); + const effects = new MockDeployEffects(); + effects.clack.inputs.push( + true, // No apps found. Do you want to create a new app? + DEPLOY_CONFIG.projectSlug, // What slug do you want to use? + "public", // Who is allowed to access your app? + true // Do you want to enable continuous deployment? + ); + + await (await open("readme.md", "a")).close(); + const {stdout, stderr} = await promisify(exec)( + "git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" + ); + console.log("starts cloud build test", {stdout, stderr}); + + await deploy(TEST_OPTIONS, effects); + + effects.close(); + }); + it("starts cloud build when continuous deployment is enabled for existing project with existing source", async () => { const deployId = "deploy123"; getCurrentObservableApi() From 159d708a38fe328919045ff13d63adb853873a6f Mon Sep 17 00:00:00 2001 From: Observable User Date: Fri, 1 Nov 2024 22:51:07 -0700 Subject: [PATCH 40/44] WHEW i think everything should be passing now, removing debug console log junk --- src/deploy.ts | 4 +--- test/deploy-test.ts | 12 +++++++----- test/mocks/directory.ts | 12 +++--------- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/src/deploy.ts b/src/deploy.ts index d135c1ca5..f7c063577 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -279,9 +279,7 @@ class Deployer { if (!isGit) throw new CliError("Not at root of a git repository."); const {ownerName, repoName} = await getGitHubRemote(); - const a = await promisify(exec)("git rev-parse --abbrev-ref HEAD"); - console.log("validateGitHubLink", {stdout: a.stdout, stderr: a.stderr}); - const branch = a.stdout.trim(); + const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout.trim(); let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); // If a source repository has already been configured, check that it’s diff --git a/test/deploy-test.ts b/test/deploy-test.ts index 5a6c82b49..2bb35af64 100644 --- a/test/deploy-test.ts +++ b/test/deploy-test.ts @@ -260,7 +260,11 @@ describe("deploy", () => { await deploy(TEST_OPTIONS, effects); assert.fail("expected error"); } catch (error) { - CliError.assert(error, {message: `Configured repository does not match local repository; check build settings on ${link(`https://observablehq.com/projects/@${DEPLOY_CONFIG.workspaceLogin}/${DEPLOY_CONFIG.projectSlug}/settings`)}`}); + CliError.assert(error, { + message: `Configured repository does not match local repository; check build settings on ${link( + `https://observablehq.com/projects/@${DEPLOY_CONFIG.workspaceLogin}/${DEPLOY_CONFIG.projectSlug}/settings` + )}` + }); } effects.close(); @@ -290,10 +294,9 @@ describe("deploy", () => { ); await (await open("readme.md", "a")).close(); - const {stdout, stderr} = await promisify(exec)( + await promisify(exec)( "git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" ); - console.log("starts cloud build test", {stdout, stderr}); await deploy(TEST_OPTIONS, effects); @@ -325,10 +328,9 @@ describe("deploy", () => { ); await (await open("readme.md", "a")).close(); - const {stdout, stderr} = await promisify(exec)( + await promisify(exec)( "git add . && git commit -m 'initial' && git remote add origin git@github.com:observablehq/test.git" ); - console.log("starts cloud build test", {stdout, stderr}); await deploy(TEST_OPTIONS, effects); diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index e79659312..b1ef82b8e 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -3,7 +3,6 @@ import {mkdtemp, rm} from "fs/promises"; import {tmpdir} from "os"; import {join} from "path/posix"; import {promisify} from "util"; -// import {rimraf} from "rimraf"; export function mockIsolatedDirectory({git}: {git: boolean}) { let dir: string; @@ -13,19 +12,14 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { dir = await mkdtemp(join(tmpdir(), "framework-test-")); process.chdir(dir); if (git) { - console.log("logging stdout, stderr"); - const a = await promisify(exec)( - 'git config --global user.email "observable@example.com" && git config --global user.name "Observable User" && git config --global init.defaultBranch main' + await promisify(exec)( + 'git config --global user.email "observable@example.com" && git config --global user.name "Observable User" && git config --global init.defaultBranch main && git init' ); - console.log(a.stdout, a.stderr); - const b = await promisify(exec)("git init"); - console.log(b.stdout, b.stderr); } }); afterEach(async () => { process.chdir(cwd); - // await rimraf(dir); - await rm(dir, {recursive: true}); //, force: true + await rm(dir, {recursive: true}); }); } From 1a5c9f9c0b5fecb6bc9240ec7996b55e60a2d53a Mon Sep 17 00:00:00 2001 From: Toph Tucker Date: Fri, 1 Nov 2024 22:54:46 -0700 Subject: [PATCH 41/44] lmao last few commits are attributed to my test user bc the exec git stuff isnt isolated so running the tests changed my email and name. happy to have you as a collaborator, Observable User. removed --global flag... duh... --- test/mocks/directory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index b1ef82b8e..edd4b961b 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -13,7 +13,7 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { process.chdir(dir); if (git) { await promisify(exec)( - 'git config --global user.email "observable@example.com" && git config --global user.name "Observable User" && git config --global init.defaultBranch main && git init' + 'git config user.email "observable@example.com" && git config user.name "Observable User" && git config init.defaultBranch main && git init' ); } }); From 4a8e4e74decf4e6b1df75735b31ec85f8de08ee0 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 1 Nov 2024 23:01:49 -0700 Subject: [PATCH 42/44] now that were not setting the git config options globally we have to init FIRST, duh duh duh --- test/mocks/directory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index edd4b961b..aaa8e3865 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -13,7 +13,7 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { process.chdir(dir); if (git) { await promisify(exec)( - 'git config user.email "observable@example.com" && git config user.name "Observable User" && git config init.defaultBranch main && git init' + 'git init && git config user.email "observable@example.com" && git config user.name "Observable User" && git config init.defaultBranch main' ); } }); From a23c35c1935c4891e7a0edfc55742026d0dfccf9 Mon Sep 17 00:00:00 2001 From: --global <--global> Date: Fri, 1 Nov 2024 23:12:32 -0700 Subject: [PATCH 43/44] set initial branch when initializing repo instead of setting the default branch after repo is already made, which of course does nothing for the current repo --- test/mocks/directory.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/mocks/directory.ts b/test/mocks/directory.ts index aaa8e3865..786938e72 100644 --- a/test/mocks/directory.ts +++ b/test/mocks/directory.ts @@ -13,7 +13,7 @@ export function mockIsolatedDirectory({git}: {git: boolean}) { process.chdir(dir); if (git) { await promisify(exec)( - 'git init && git config user.email "observable@example.com" && git config user.name "Observable User" && git config init.defaultBranch main' + 'git init -b main && git config user.email "observable@example.com" && git config user.name "Observable User"' ); } }); From 3e5e6691dbb0c5d487f235482e7e2b3e1751b658 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Rivi=C3=A8re?= Date: Tue, 12 Nov 2024 02:00:46 +0100 Subject: [PATCH 44/44] Fil/onramp review (#1805) * documentation * reformat and type * simpler * clarify the logic in validateGitHubLink fixes an uncaught Error ('Setting source repository for continuous deployment failed' was never reached) --- docs/deploying.md | 14 +-- src/deploy.ts | 180 +++++++++++++++---------------------- src/observableApiClient.ts | 30 +++---- 3 files changed, 92 insertions(+), 132 deletions(-) diff --git a/docs/deploying.md b/docs/deploying.md index 7113a0811..043127235 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -38,19 +38,19 @@ npm run deploy -- --help ## Continuous deployment - +### Cloud builds -You can connect your app to Observable to handle deploys automatically. You can automate deploys both [on commit](https://observablehq.com/documentation/data-apps/github) (whenever you push a new commit to your project’s default branch) and [on schedule](https://observablehq.com/documentation/data-apps/schedules) (such as daily or weekly). +Connect your app to Observable to handle deploys automatically. You can automate deploys both [on commit](https://observablehq.com/documentation/data-apps/github) (whenever you push a new commit to your project’s default branch) and [on schedule](https://observablehq.com/documentation/data-apps/schedules) (such as daily or weekly). -Automatic deploys — also called _continuous deployment_ or _CD_ — ensure that your data is always up to date, and that any changes you make to your app are immediately reflected in the deployed version. +Continuous deployment (for short, _CD_) ensures that your data is always up to date, and that any changes you make to your app are immediately reflected in the deployed version. On your app settings page on Observable, open the **Build settings** tab to set up a link to a GitHub repository hosting your project’s files. Observable will then listen for changes in the repo and deploy the app automatically. -The settings page also allows you to trigger a manual deploy on Observable Cloud, add secrets (for data loaders to use private APIs and passwords), view logs, configure sharing, _etc._ For details, see the [Building & deploying](https://observablehq.com/documentation/data-apps/deploys) documentation. +The settings page also allows you to trigger a manual deploy, add secrets for data loaders to use private APIs and passwords, view logs, configure sharing, _etc._ For details, see the [Building & deploying](https://observablehq.com/documentation/data-apps/deploys) documentation. -## GitHub Actions +### GitHub Actions -As an alternative to building on Observable Cloud, you can use [GitHub Actions](https://github.com/features/actions) and have GitHub build a new version of your app and deploy it to Observable. In your git repository, create and commit a file at `.github/workflows/deploy.yml`. Here is a starting example: +Alternatively, you can use [GitHub Actions](https://github.com/features/actions) to have GitHub build a new version of your app and deploy it to Observable. In your git repository, create and commit a file at `.github/workflows/deploy.yml`. Here is a starting example: ```yaml name: Deploy @@ -139,6 +139,8 @@ This uses one cache per calendar day (in the `America/Los_Angeles` time zone). I
You’ll need to edit the paths above if you’ve configured a source root other than src.
+
Caching is limited for now to manual builds and GitHub Actions. In the future, it will be available as a configuration option for Observable Cloud builds.
+ ## Deploy configuration The deploy command creates a file at .observablehq/deploy.json under the source root (typically src) with information on where to deploy the app. This file allows you to re-deploy an app without having to repeat where you want the app to live on Observable. diff --git a/src/deploy.ts b/src/deploy.ts index f7c063577..739fae594 100644 --- a/src/deploy.ts +++ b/src/deploy.ts @@ -39,41 +39,21 @@ const BUILD_AGE_WARNING_MS = 1000 * 60 * 5; const OBSERVABLE_UI_ORIGIN = getObservableUiOrigin(); function settingsUrl(deployTarget: DeployTargetInfo) { - if (deployTarget.create) { - throw new Error("Incorrect deploy target state"); - } + if (deployTarget.create) throw new Error("Incorrect deploy target state"); return `${OBSERVABLE_UI_ORIGIN}projects/@${deployTarget.workspace.login}/${deployTarget.project.slug}`; } /** * Returns the ownerName and repoName of the first GitHub remote (HTTPS or SSH) - * on the current repository, or null. + * on the current repository. Supports both https and ssh URLs: + * - https://github.com/observablehq/framework.git + * - git@github.com:observablehq/framework.git */ -async function getGitHubRemote() { - const remotes = (await promisify(exec)("git remote -v")).stdout - .split("\n") - .filter((d) => d) - .map((d) => { - const [, url] = d.split(/\s/g); - if (url.startsWith("https://github.com/")) { - // HTTPS: https://github.com/observablehq/framework.git - const [ownerName, repoName] = new URL(url).pathname - .slice(1) - .replace(/\.git$/, "") - .split("/"); - return {ownerName, repoName}; - } else if (url.startsWith("git@github.com:")) { - // SSH: git@github.com:observablehq/framework.git - const [ownerName, repoName] = url - .replace(/^git@github.com:/, "") - .replace(/\.git$/, "") - .split("/"); - return {ownerName, repoName}; - } - }); - const remote = remotes.find((d) => d && d.ownerName && d.repoName); - if (!remote) throw new CliError("No GitHub remote found."); - return remote ?? null; +async function getGitHubRemote(): Promise<{ownerName: string; repoName: string} | undefined> { + const firstRemote = (await promisify(exec)("git remote -v")).stdout.match( + /^\S+\s(https:\/\/github.com\/|git@github.com:)(?[^/]+)\/(?[^/]*?)(\.git)?\s/m + ); + return firstRemote?.groups as {ownerName: string; repoName: string} | undefined; } export interface DeployOptions { @@ -223,20 +203,13 @@ class Deployer { const {deployId} = this.deployOptions; if (!deployId) throw new Error("invalid deploy options"); await this.checkDeployCreated(deployId); - - const buildFilePaths = await this.getBuildFilePaths(); - - await this.uploadFiles(deployId, buildFilePaths); + await this.uploadFiles(deployId, await this.getBuildFilePaths()); await this.markDeployUploaded(deployId); - const deployInfo = await this.pollForProcessingCompletion(deployId); - - return deployInfo; + return await this.pollForProcessingCompletion(deployId); } private async cloudBuild(deployTarget: DeployTargetInfo) { - if (deployTarget.create) { - throw new Error("Incorrect deploy target state"); - } + if (deployTarget.create) throw new Error("Incorrect deploy target state"); const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; await this.apiClient.postProjectBuild(deployTarget.project.id); const spinner = this.effects.clack.spinner(); @@ -264,51 +237,46 @@ class Deployer { } } - // Throws error if local and remote GitHub repos don’t match or are invalid + // Throws error if local and remote GitHub repos don’t match or are invalid. + // Ignores this.deployOptions.config.root as we only support cloud builds from + // the root directory. private async validateGitHubLink(deployTarget: DeployTargetInfo): Promise { - if (deployTarget.create) { - throw new Error("Incorrect deploy target state"); - } - if (!deployTarget.project.build_environment_id) { - // TODO: allow setting build environment from CLI - throw new CliError("No build environment configured."); - } - // We only support cloud builds from the root directory so this ignores - // this.deployOptions.config.root - const isGit = existsSync(".git"); - if (!isGit) throw new CliError("Not at root of a git repository."); - - const {ownerName, repoName} = await getGitHubRemote(); + if (deployTarget.create) throw new Error("Incorrect deploy target state"); + if (!deployTarget.project.build_environment_id) throw new CliError("No build environment configured."); + if (!existsSync(".git")) throw new CliError("Not at root of a git repository."); + const remote = await getGitHubRemote(); + if (!remote) throw new CliError("No GitHub remote found."); const branch = (await promisify(exec)("git rev-parse --abbrev-ref HEAD")).stdout.trim(); - let localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); + if (!branch) throw new Error("Branch not found."); // If a source repository has already been configured, check that it’s - // accessible and matches the local repository and branch. - // TODO: validate local/remote refs match, "Your branch is up to date", - // and "nothing to commit, working tree clean". + // accessible and matches the linked repository and branch. TODO: validate + // local/remote refs match, "Your branch is up to date", and "nothing to + // commit, working tree clean". const {source} = deployTarget.project; if (source) { - if (localRepo && source.provider_id !== localRepo.provider_id) { - throw new CliError( - `Configured repository does not match local repository; check build settings on ${link( - `${settingsUrl(deployTarget)}/settings` - )}` - ); - } - if (localRepo && source.branch && source.branch !== branch) { - // TODO: If source.branch is empty, it'll use the default repository - // branch (usually main or master), which we don't know from our current - // getGitHubRepository response, and thus can't check here. - throw new CliError( - `Configured branch does not match local branch; check build settings on ${link( - `${settingsUrl(deployTarget)}/settings` - )}` - ); + const linkedRepo = await this.apiClient.getGitHubRepository(remote); + if (linkedRepo) { + if (source.provider_id !== linkedRepo.provider_id) { + throw new CliError( + `Configured repository does not match local repository; check build settings on ${link( + `${settingsUrl(deployTarget)}/settings` + )}` + ); + } + if (source.branch !== branch) { + // TODO: If source.branch is empty, it'll use the default repository + // branch (usually main or master), which we don't know from our current + // getGitHubRepository response, and thus can't check here. + throw new CliError( + `Configured branch ${source.branch} does not match local branch ${branch}; check build settings on ${link( + `${settingsUrl(deployTarget)}/settings` + )}` + ); + } } - const remoteAuthedRepo = await this.apiClient.getGitHubRepository({ - providerId: source.provider_id - }); - if (!remoteAuthedRepo) { + + if (!(await this.apiClient.getGitHubRepository({providerId: source.provider_id}))) { // TODO: This could poll for auth too, but is a distinct case because it // means the repo was linked at one point and then something went wrong throw new CliError( @@ -322,7 +290,10 @@ class Deployer { return; } - if (!localRepo) { + // If the source has not been configured, first check that the remote repo + // is linked in CD settings. If not, prompt the user to auth & link. + let linkedRepo = await this.apiClient.getGitHubRepository(remote); + if (!linkedRepo) { if (!this.effects.isTty) throw new CliError( "Cannot access repository for continuous deployment and cannot request access in non-interactive mode" @@ -330,43 +301,38 @@ class Deployer { // Repo is not authorized; link to auth page and poll for auth const authUrl = new URL("/auth-github", OBSERVABLE_UI_ORIGIN); - authUrl.searchParams.set("owner", ownerName); - authUrl.searchParams.set("repo", repoName); - this.effects.clack.log.info(`Authorize Observable to access the ${bold(repoName)} repository: ${link(authUrl)}`); + authUrl.searchParams.set("owner", remote.ownerName); + authUrl.searchParams.set("repo", remote.repoName); + this.effects.clack.log.info( + `Authorize Observable to access the ${bold(remote.repoName)} repository: ${link(authUrl)}` + ); const spinner = this.effects.clack.spinner(); - spinner.start("Waiting for repository to be authorized"); + spinner.start("Waiting for authorization"); const {deployPollInterval: pollInterval = DEPLOY_POLL_INTERVAL_MS} = this.deployOptions; const pollExpiration = Date.now() + DEPLOY_POLL_MAX_MS; - while (!localRepo) { + do { await new Promise((resolve) => setTimeout(resolve, pollInterval)); if (Date.now() > pollExpiration) { - spinner.stop("Waiting for repository to be authorized timed out."); + spinner.stop("Authorization timed out."); throw new CliError("Repository authorization failed"); } - localRepo = await this.apiClient.getGitHubRepository({ownerName, repoName}); - if (localRepo) spinner.stop("Repository authorized."); - } + } while (!(linkedRepo = await this.apiClient.getGitHubRepository(remote))); + spinner.stop("Repository authorized."); } - const response = await this.apiClient.postProjectEnvironment(deployTarget.project.id, { - source: { - provider: localRepo.provider, - provider_id: localRepo.provider_id, - url: localRepo.url, - branch - } - }); - - if (!response) throw new CliError("Setting source repository for continuous deployment failed"); - - // Configured repo is OK; proceed - return; + // Save the linked repo as the configured source. + const {provider, provider_id, url} = linkedRepo; + await this.apiClient + .postProjectEnvironment(deployTarget.project.id, {source: {provider, provider_id, url, branch}}) + .catch((error) => { + throw new CliError("Setting source repository for continuous deployment failed", {cause: error}); + }); } private async startNewDeploy(): Promise { const {deployConfig, deployTarget} = await this.getDeployTarget(await this.getUpdatedDeployConfig()); - let deployId: string | null; + let deployId: string; if (deployConfig.continuousDeployment) { await this.validateGitHubLink(deployTarget); deployId = await this.cloudBuild(deployTarget); @@ -388,11 +354,7 @@ class Deployer { } return deployInfo; } catch (error) { - if (isHttpError(error)) { - throw new CliError(`Deploy ${deployId} not found.`, { - cause: error - }); - } + if (isHttpError(error)) throw new CliError(`Deploy ${deployId} not found.`, {cause: error}); throw error; } } @@ -580,7 +542,7 @@ class Deployer { continuousDeployment = enable; } - const newDeployConfig = { + deployConfig = { projectId: deployTarget.project.id, projectSlug: deployTarget.project.slug, workspaceLogin: deployTarget.workspace.login, @@ -590,11 +552,11 @@ class Deployer { await this.effects.setDeployConfig( this.deployOptions.config.root, this.deployOptions.deployConfigPath, - newDeployConfig, + deployConfig, this.effects ); - return {deployConfig: newDeployConfig, deployTarget}; + return {deployConfig, deployTarget}; } // Create the new deploy on the server. diff --git a/src/observableApiClient.ts b/src/observableApiClient.ts index 5f48a2778..7ea717061 100644 --- a/src/observableApiClient.ts +++ b/src/observableApiClient.ts @@ -130,22 +130,20 @@ export class ObservableApiClient { async getGitHubRepository( props: {ownerName: string; repoName: string} | {providerId: string} ): Promise { - let url: URL; - if ("providerId" in props) { - url = new URL(`/cli/github/repository?provider_id=${props.providerId}`, this._apiOrigin); - } else { - url = new URL(`/cli/github/repository?owner=${props.ownerName}&repo=${props.repoName}`, this._apiOrigin); - } - try { - return await this._fetch(url, {method: "GET"}); - } catch (err) { - // TODO: err.details.errors may be [{code: "NO_GITHUB_TOKEN"}] or [{code: "NO_REPO_ACCESS"}], - // which could be handled separately - return null; - } + const params = + "providerId" in props ? `provider_id=${props.providerId}` : `owner=${props.ownerName}&repo=${props.repoName}`; + return await this._fetch( + new URL(`/cli/github/repository?${params}`, this._apiOrigin), + {method: "GET"} + ).catch(() => null); + // TODO: err.details.errors may be [{code: "NO_GITHUB_TOKEN"}] or [{code: "NO_REPO_ACCESS"}], + // which could be handled separately } - async postProjectEnvironment(id, body): Promise { + async postProjectEnvironment( + id: string, + body: {source: {provider: "github"; provider_id: string; url: string; branch: string}} + ): Promise { const url = new URL(`/cli/project/${id}/environment`, this._apiOrigin); return await this._fetch(url, { method: "POST", @@ -155,9 +153,7 @@ export class ObservableApiClient { } async postProjectBuild(id): Promise<{id: string}> { - return await this._fetch<{id: string}>(new URL(`/cli/project/${id}/build`, this._apiOrigin), { - method: "POST" - }); + return await this._fetch<{id: string}>(new URL(`/cli/project/${id}/build`, this._apiOrigin), {method: "POST"}); } async postProject({