From 716912cb8fdb061515d7a9da83612f624c80774b Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 12 Jan 2024 17:41:20 +0500 Subject: [PATCH 01/17] update configuration options --- config.yml | 12 +++++--- docker-compose.yml | 16 ++++++++--- package.json | 3 +- src/schemas/config.ts | 67 ++++++++++++++++++++++++++++++------------- 4 files changed, 69 insertions(+), 29 deletions(-) diff --git a/config.yml b/config.yml index 959d0993..67fc6c89 100644 --- a/config.yml +++ b/config.yml @@ -34,10 +34,14 @@ logs: # username: elastic-username # password: elastic-password youtube: - clientId: google-client-id - clientSecret: google-client-secret - # adcKeyFilePath: path/to/adc-key-file.json - # maxAllowedQuotaUsageInPercentage: 95 + apiMode: api + api: + clientId: google-client-id + clientSecret: google-client-secret + # adcKeyFilePath: path/to/adc-key-file.json + # maxAllowedQuotaUsageInPercentage: 95 + operationalApi: + url: http://127.0.0.1:8889 proxy: url: socks5://chisel-client:1080 chiselProxy: diff --git a/docker-compose.yml b/docker-compose.yml index ec5f0af4..882d5b08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ services: - ./local/logs:/youtube-synch/local/logs - ./local/data:/youtube-synch/local/data # mount Google Cloud's Application Default Credentials file. A default bind mount is created - # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined, + # as workaround for scenario when `YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH` will be undefined, # since docker-compose does not support creating bind-mount volume with empty path. - ${YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH:-./package.json}:${YT_SYNCH__YOUTUBE__ADC_KEY_FILE_PATH:-/package.json}:ro # mount the AWS credentials file to docker container home directory @@ -30,8 +30,9 @@ services: YT_SYNCH__ENDPOINTS__REDIS__PORT: ${YT_SYNCH__ENDPOINTS__REDIS__PORT} # YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT} # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY} - # YT_SYNCH__YOUTUBE__CLIENT_ID: ${YT_SYNCH__YOUTUBE__CLIENT_ID} - # YT_SYNCH__YOUTUBE__CLIENT_SECRET: ${YT_SYNCH__YOUTUBE__CLIENT_SECRET} + YT_SYNCH__YOUTUBE__OPERATIONAL_API__URL: http://youtube-operational-api:8889 + # YT_SYNCH__YOUTUBE__API__CLIENT_ID: ${YT_SYNCH__YOUTUBE__API__CLIENT_ID} + # YT_SYNCH__YOUTUBE__API__CLIENT_SECRET: ${YT_SYNCH__YOUTUBE__API__CLIENT_SECRET} # YT_SYNCH__AWS__ENDPOINT: ${YT_SYNCH__AWS__ENDPOINT} # YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID: ${YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID} # YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY: ${YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY} @@ -48,13 +49,20 @@ services: redis: image: redis:7.2.1 container_name: redis - command: [ "redis-server", "--maxmemory-policy", "noeviction" ] + command: ['redis-server', '--maxmemory-policy', 'noeviction'] ports: - 127.0.0.1:${YT_SYNCH__ENDPOINTS__REDIS__PORT}:${YT_SYNCH__ENDPOINTS__REDIS__PORT} networks: - youtube-synch volumes: - redis-data:/data + youtube-operational-api: + image: benjaminloison/youtube-operational-api + container_name: youtube-operational-api + ports: + - 127.0.0.1:8889:80 + networks: + - youtube-synch volumes: logs: diff --git a/package.json b/package.json index a095df6d..c17aaa1f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "youtube-sync", - "version": "3.3.0", + "version": "4.0.0", "license": "MIT", "scripts": { "postpack": "rm -f oclif.manifest.json", @@ -63,6 +63,7 @@ "@typescript-eslint/eslint-plugin": "^5.54.0", "@typescript-eslint/parser": "^5.54.0", "eslint": "8.2.0", + "get-graphql-schema": "^2.1.2", "prettier": "^2.5.1", "ts-node": "^10.9.1", "typescript": "4.5.2" diff --git a/src/schemas/config.ts b/src/schemas/config.ts index 61c2f207..4a182e40 100644 --- a/src/schemas/config.ts +++ b/src/schemas/config.ts @@ -172,30 +172,57 @@ export const configSchema: JSONSchema7 = objectSchema({ required: [], }), youtube: objectSchema({ - title: 'Youtube Oauth2 Client configuration', - description: 'Youtube Oauth2 Client configuration', + title: 'Youtube related configuration', + description: 'Youtube related configuration', properties: { - clientId: { type: 'string', description: 'Youtube Oauth2 Client Id' }, - clientSecret: { type: 'string', description: 'Youtube Oauth2 Client Secret' }, - maxAllowedQuotaUsageInPercentage: { - description: - `Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. ` + - `Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). ` + - `All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups.`, - type: 'number', - default: 95, - }, - adcKeyFilePath: { - type: 'string', + apiMode: { type: 'string', enum: ['api-free', 'api', 'both'], default: 'both' }, + api: objectSchema({ + title: 'Youtube API configuration', + description: 'Youtube API configuration', + properties: { + clientId: { type: 'string', description: 'Youtube Oauth2 Client Id' }, + clientSecret: { type: 'string', description: 'Youtube Oauth2 Client Secret' }, + maxAllowedQuotaUsageInPercentage: { + description: + `Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. ` + + `Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). ` + + `All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups.`, + type: 'number', + default: 95, + }, + adcKeyFilePath: { + type: 'string', + description: + `Path to the Google Cloud's Application Default Credentials (ADC) key file. ` + + `It is required to periodically monitor the Youtube API quota usage.`, + }, + }, + required: ['clientId', 'clientSecret'], + dependencies: { + maxAllowedQuotaUsageInPercentage: ['adcKeyFilePath'], + }, + }), + operationalApi: objectSchema({ + title: 'Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration', description: - `Path to the Google Cloud's Application Default Credentials (ADC) key file. ` + - `It is required to periodically monitor the Youtube API quota usage.`, - }, + 'Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration', + properties: { + url: { + type: 'string', + description: 'URL of the Youtube Operational API server (for example: http://localhost:8080)', + }, + }, + required: ['url'], + }), }, - dependencies: { - maxAllowedQuotaUsageInPercentage: ['adcKeyFilePath'], + + if: { + properties: { apiMode: { enum: ['api', 'both'] } }, + }, + then: { + required: ['api'], }, - required: ['clientId', 'clientSecret'], + required: ['apiMode', 'operationalApi'], }), aws: objectSchema({ title: 'AWS configurations needed to connect with DynamoDB instance', From 385a38e91d8012eb6971bdda8055bb7c2c0bb935 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 12 Jan 2024 17:42:01 +0500 Subject: [PATCH 02/17] update config docs --- ...ted-configuration-if-properties-apimode.md | 12 ++ ...-configuration-if-properties-signupmode.md | 12 ++ ...ube-related-configuration-if-properties.md | 3 + ...erties-youtube-related-configuration-if.md | 36 +++++ ...elated-configuration-properties-apimode.md | 21 +++ ...ted-configuration-properties-signupmode.md | 21 +++ ...-youtube-api-configuration-dependencies.md | 3 + ...configuration-properties-adckeyfilepath.md | 3 + ...e-api-configuration-properties-clientid.md | 3 + ...i-configuration-properties-clientsecret.md | 3 + ...erties-maxallowedquotausageinpercentage.md | 11 ++ ...on-properties-youtube-api-configuration.md | 92 +++++++++++++ ...tional-api-configuration-properties-url.md | 3 + ...onyoutube-operational-api-configuration.md | 27 ++++ ...ties-youtube-related-configuration-then.md | 3 + ...roperties-youtube-related-configuration.md | 83 +++++++++++ ...ding-related-configuration-dependencies.md | 3 + ...-configuration-if-properties-signupmode.md | 12 ++ ...ing-related-configuration-if-properties.md | 3 + ...gnuponboarding-related-configuration-if.md | 36 +++++ ...configuration-properties-adckeyfilepath.md | 3 + ...lated-configuration-properties-clientid.md | 3 + ...d-configuration-properties-clientsecret.md | 3 + ...erties-maxallowedquotausageinpercentage.md | 11 ++ ...ted-configuration-properties-signupmode.md | 21 +++ ...uponboarding-related-configuration-then.md | 3 + ...-signuponboarding-related-configuration.md | 129 ++++++++++++++++++ docs/config/definition.md | 10 +- 28 files changed, 568 insertions(+), 5 deletions(-) create mode 100644 docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-if-properties.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-if.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-apimode.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration-properties-url.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md create mode 100644 docs/config/definition-properties-youtube-related-configuration-then.md create mode 100644 docs/config/definition-properties-youtube-related-configuration.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md create mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration.md diff --git a/docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md b/docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md new file mode 100644 index 00000000..646e3cc5 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md @@ -0,0 +1,12 @@ +## apiMode Type + +unknown + +## apiMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :------- | :---------- | +| `"api"` | | +| `"both"` | | diff --git a/docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md b/docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md new file mode 100644 index 00000000..4bd91053 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md @@ -0,0 +1,12 @@ +## signupMode Type + +unknown + +## signupMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :------- | :---------- | +| `"api"` | | +| `"both"` | | diff --git a/docs/config/definition-properties-youtube-related-configuration-if-properties.md b/docs/config/definition-properties-youtube-related-configuration-if-properties.md new file mode 100644 index 00000000..c89940c6 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-if-properties.md @@ -0,0 +1,3 @@ +## properties Type + +unknown diff --git a/docs/config/definition-properties-youtube-related-configuration-if.md b/docs/config/definition-properties-youtube-related-configuration-if.md new file mode 100644 index 00000000..00a9a02b --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-if.md @@ -0,0 +1,36 @@ +## if Type + +unknown + +# if Properties + +| Property | Type | Required | Nullable | Defined by | +| :------------------ | :------------ | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [apiMode](#apimode) | Not specified | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-if-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/apiMode") | + +## apiMode + + + +`apiMode` + +* is optional + +* Type: unknown + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-if-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/apiMode") + +### apiMode Type + +unknown + +### apiMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :------- | :---------- | +| `"api"` | | +| `"both"` | | diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-apimode.md b/docs/config/definition-properties-youtube-related-configuration-properties-apimode.md new file mode 100644 index 00000000..06e5445e --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-apimode.md @@ -0,0 +1,21 @@ +## apiMode Type + +`string` + +## apiMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :----------- | :---------- | +| `"api-free"` | | +| `"api"` | | +| `"both"` | | + +## apiMode Default Value + +The default value is: + +```json +"both" +``` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md b/docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md new file mode 100644 index 00000000..c053e307 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md @@ -0,0 +1,21 @@ +## signupMode Type + +`string` + +## signupMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :----------- | :---------- | +| `"api-free"` | | +| `"api"` | | +| `"both"` | | + +## signupMode Default Value + +The default value is: + +```json +"both" +``` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md new file mode 100644 index 00000000..5fece311 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md @@ -0,0 +1,3 @@ +## dependencies Type + +unknown diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md new file mode 100644 index 00000000..b029962a --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md @@ -0,0 +1,3 @@ +## adcKeyFilePath Type + +`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md new file mode 100644 index 00000000..a7b5d5fb --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md @@ -0,0 +1,3 @@ +## clientId Type + +`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md new file mode 100644 index 00000000..53452148 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md @@ -0,0 +1,3 @@ +## clientSecret Type + +`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md new file mode 100644 index 00000000..7c692483 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md @@ -0,0 +1,11 @@ +## maxAllowedQuotaUsageInPercentage Type + +`number` + +## maxAllowedQuotaUsageInPercentage Default Value + +The default value is: + +```json +95 +``` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md new file mode 100644 index 00000000..59a856b6 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md @@ -0,0 +1,92 @@ +## api Type + +`object` ([Youtube API configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md)) + +# api Properties + +| Property | Type | Required | Nullable | Defined by | +| :-------------------------------------------------------------------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [clientId](#clientid) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientId") | +| [clientSecret](#clientsecret) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientSecret") | +| [maxAllowedQuotaUsageInPercentage](#maxallowedquotausageinpercentage) | `number` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/maxAllowedQuotaUsageInPercentage") | +| [adcKeyFilePath](#adckeyfilepath) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/adcKeyFilePath") | + +## clientId + +Youtube Oauth2 Client Id + +`clientId` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientId") + +### clientId Type + +`string` + +## clientSecret + +Youtube Oauth2 Client Secret + +`clientSecret` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientSecret") + +### clientSecret Type + +`string` + +## maxAllowedQuotaUsageInPercentage + +Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups. + +`maxAllowedQuotaUsageInPercentage` + +* is optional + +* Type: `number` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/maxAllowedQuotaUsageInPercentage") + +### maxAllowedQuotaUsageInPercentage Type + +`number` + +### maxAllowedQuotaUsageInPercentage Default Value + +The default value is: + +```json +95 +``` + +## adcKeyFilePath + +Path to the Google Cloud's Application Default Credentials (ADC) key file. It is required to periodically monitor the Youtube API quota usage. + +`adcKeyFilePath` + +* is optional + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/adcKeyFilePath") + +### adcKeyFilePath Type + +`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration-properties-url.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration-properties-url.md new file mode 100644 index 00000000..073bf9cf --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration-properties-url.md @@ -0,0 +1,3 @@ +## url Type + +`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md new file mode 100644 index 00000000..f098361c --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md @@ -0,0 +1,27 @@ +## operationalApi Type + +`object` ([Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md)) + +# operationalApi Properties + +| Property | Type | Required | Nullable | Defined by | +| :---------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [url](#url) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration-properties-url.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/operationalApi/properties/url") | + +## url + +URL of the Youtube Operational API server (for example: ) + +`url` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration-properties-url.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/operationalApi/properties/url") + +### url Type + +`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-then.md b/docs/config/definition-properties-youtube-related-configuration-then.md new file mode 100644 index 00000000..139b21d9 --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration-then.md @@ -0,0 +1,3 @@ +## then Type + +unknown diff --git a/docs/config/definition-properties-youtube-related-configuration.md b/docs/config/definition-properties-youtube-related-configuration.md new file mode 100644 index 00000000..5ab91f8e --- /dev/null +++ b/docs/config/definition-properties-youtube-related-configuration.md @@ -0,0 +1,83 @@ +## youtube Type + +`object` ([Youtube related configuration](definition-properties-youtube-related-configuration.md)) + +# youtube Properties + +| Property | Type | Required | Nullable | Defined by | +| :-------------------------------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [apiMode](#apimode) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/apiMode") | +| [api](#api) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api") | +| [operationalApi](#operationalapi) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/operationalApi") | + +## apiMode + + + +`apiMode` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/apiMode") + +### apiMode Type + +`string` + +### apiMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :----------- | :---------- | +| `"api-free"` | | +| `"api"` | | +| `"both"` | | + +### apiMode Default Value + +The default value is: + +```json +"both" +``` + +## api + +Youtube API configuration + +`api` + +* is optional + +* Type: `object` ([Youtube API configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api") + +### api Type + +`object` ([Youtube API configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md)) + +## operationalApi + +Youtube Operational API () configuration + +`operationalApi` + +* is required + +* Type: `object` ([Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md)) + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/operationalApi") + +### operationalApi Type + +`object` ([Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md)) diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md new file mode 100644 index 00000000..5fece311 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md @@ -0,0 +1,3 @@ +## dependencies Type + +unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md new file mode 100644 index 00000000..4bd91053 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md @@ -0,0 +1,12 @@ +## signupMode Type + +unknown + +## signupMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :------- | :---------- | +| `"api"` | | +| `"both"` | | diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md new file mode 100644 index 00000000..c89940c6 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md @@ -0,0 +1,3 @@ +## properties Type + +unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md new file mode 100644 index 00000000..9af00676 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md @@ -0,0 +1,36 @@ +## if Type + +unknown + +# if Properties + +| Property | Type | Required | Nullable | Defined by | +| :------------------------ | :------------ | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| [signupMode](#signupmode) | Not specified | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/signupMode") | + +## signupMode + + + +`signupMode` + +* is optional + +* Type: unknown + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/signupMode") + +### signupMode Type + +unknown + +### signupMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :------- | :---------- | +| `"api"` | | +| `"both"` | | diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md new file mode 100644 index 00000000..b029962a --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md @@ -0,0 +1,3 @@ +## adcKeyFilePath Type + +`string` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md new file mode 100644 index 00000000..a7b5d5fb --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md @@ -0,0 +1,3 @@ +## clientId Type + +`string` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md new file mode 100644 index 00000000..53452148 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md @@ -0,0 +1,3 @@ +## clientSecret Type + +`string` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md new file mode 100644 index 00000000..7c692483 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md @@ -0,0 +1,11 @@ +## maxAllowedQuotaUsageInPercentage Type + +`number` + +## maxAllowedQuotaUsageInPercentage Default Value + +The default value is: + +```json +95 +``` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md new file mode 100644 index 00000000..c053e307 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md @@ -0,0 +1,21 @@ +## signupMode Type + +`string` + +## signupMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :----------- | :---------- | +| `"api-free"` | | +| `"api"` | | +| `"both"` | | + +## signupMode Default Value + +The default value is: + +```json +"both" +``` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md new file mode 100644 index 00000000..139b21d9 --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md @@ -0,0 +1,3 @@ +## then Type + +unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration.md new file mode 100644 index 00000000..8f1778ce --- /dev/null +++ b/docs/config/definition-properties-youtube-signuponboarding-related-configuration.md @@ -0,0 +1,129 @@ +## youtube Type + +`object` ([Youtube Signup/Onboarding related configuration](definition-properties-youtube-signuponboarding-related-configuration.md)) + +# youtube Properties + +| Property | Type | Required | Nullable | Defined by | +| :-------------------------------------------------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [signupMode](#signupmode) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/signupMode") | +| [clientId](#clientid) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientId") | +| [clientSecret](#clientsecret) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientSecret") | +| [maxAllowedQuotaUsageInPercentage](#maxallowedquotausageinpercentage) | `number` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/maxAllowedQuotaUsageInPercentage") | +| [adcKeyFilePath](#adckeyfilepath) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/adcKeyFilePath") | + +## signupMode + + + +`signupMode` + +* is required + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/signupMode") + +### signupMode Type + +`string` + +### signupMode Constraints + +**enum**: the value of this property must be equal to one of the following values: + +| Value | Explanation | +| :----------- | :---------- | +| `"api-free"` | | +| `"api"` | | +| `"both"` | | + +### signupMode Default Value + +The default value is: + +```json +"both" +``` + +## clientId + +Youtube Oauth2 Client Id + +`clientId` + +* is optional + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientId") + +### clientId Type + +`string` + +## clientSecret + +Youtube Oauth2 Client Secret + +`clientSecret` + +* is optional + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientSecret") + +### clientSecret Type + +`string` + +## maxAllowedQuotaUsageInPercentage + +Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups. + +`maxAllowedQuotaUsageInPercentage` + +* is optional + +* Type: `number` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/maxAllowedQuotaUsageInPercentage") + +### maxAllowedQuotaUsageInPercentage Type + +`number` + +### maxAllowedQuotaUsageInPercentage Default Value + +The default value is: + +```json +95 +``` + +## adcKeyFilePath + +Path to the Google Cloud's Application Default Credentials (ADC) key file. It is required to periodically monitor the Youtube API quota usage. + +`adcKeyFilePath` + +* is optional + +* Type: `string` + +* cannot be null + +* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/adcKeyFilePath") + +### adcKeyFilePath Type + +`string` diff --git a/docs/config/definition.md b/docs/config/definition.md index b252d076..d5e42611 100644 --- a/docs/config/definition.md +++ b/docs/config/definition.md @@ -9,7 +9,7 @@ | [joystream](#joystream) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream") | | [endpoints](#endpoints) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-endpoints.md "https://joystream.org/schemas/youtube-synch/config#/properties/endpoints") | | [logs](#logs) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-logs.md "https://joystream.org/schemas/youtube-synch/config#/properties/logs") | -| [youtube](#youtube) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube") | +| [youtube](#youtube) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube") | | [aws](#aws) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-aws-configurations-needed-to-connect-with-dynamodb-instance.md "https://joystream.org/schemas/youtube-synch/config#/properties/aws") | | [proxy](#proxy) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-socks5-proxy-client-configuration-used-by-yt-dlp-to-bypass-ip-blockage-by-youtube.md "https://joystream.org/schemas/youtube-synch/config#/properties/proxy") | | [creatorOnboardingRequirements](#creatoronboardingrequirements) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-creatoronboardingrequirements.md "https://joystream.org/schemas/youtube-synch/config#/properties/creatorOnboardingRequirements") | @@ -72,21 +72,21 @@ Specifies the logging configuration ## youtube -Youtube Oauth2 Client configuration +Youtube related configuration `youtube` * is required -* Type: `object` ([Youtube Oauth2 Client configuration](definition-properties-youtube-oauth2-client-configuration.md)) +* Type: `object` ([Youtube related configuration](definition-properties-youtube-related-configuration.md)) * cannot be null -* defined in: [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube") +* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube") ### youtube Type -`object` ([Youtube Oauth2 Client configuration](definition-properties-youtube-oauth2-client-configuration.md)) +`object` ([Youtube related configuration](definition-properties-youtube-related-configuration.md)) ## aws From 8fefd9d22327e1418068109e87259002a43fb3b8 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 12 Jan 2024 17:45:17 +0500 Subject: [PATCH 03/17] refactor Youtube API services and integrate with Youtube-Operatiional-API --- src/services/youtube/api.ts | 253 +++++-------------------- src/services/youtube/index.ts | 25 +++ src/services/youtube/openApi.ts | 179 +++++++++++++++++ src/services/youtube/operationalApi.ts | 211 +++++++++++++++++++++ 4 files changed, 461 insertions(+), 207 deletions(-) create mode 100644 src/services/youtube/index.ts create mode 100644 src/services/youtube/openApi.ts create mode 100644 src/services/youtube/operationalApi.ts diff --git a/src/services/youtube/api.ts b/src/services/youtube/api.ts index e65e5d70..b4ad4edf 100644 --- a/src/services/youtube/api.ts +++ b/src/services/youtube/api.ts @@ -1,7 +1,5 @@ -import ffmpegInstaller from '@ffmpeg-installer/ffmpeg' import { MetricServiceClient } from '@google-cloud/monitoring' import { youtube_v3 } from '@googleapis/youtube' -import { exec } from 'child_process' import { OAuth2Client } from 'google-auth-library' import { GetTokenResponse } from 'google-auth-library/build/src/auth/oauth2client' import { GaxiosError } from 'googleapis-common' @@ -10,187 +8,51 @@ import _ from 'lodash' import moment from 'moment-timezone' import { FetchError } from 'node-fetch' import path from 'path' -import pkgDir from 'pkg-dir' -import { promisify } from 'util' -import ytdl from 'youtube-dl-exec' import { StatsRepository } from '../../repository' import { ReadonlyConfig, WithRequired, formattedJSON } from '../../types' import { ExitCodes, YoutubeApiError } from '../../types/errors' -import { YtChannel, YtDlpFlatPlaylistOutput, YtDlpVideoOutput, YtUser, YtVideo } from '../../types/youtube' +import { YtChannel, YtUser, YtVideo } from '../../types/youtube' +import { YtDlpClient } from './openApi' import Schema$Video = youtube_v3.Schema$Video import Schema$Channel = youtube_v3.Schema$Channel -export interface IOpenYTApi { - ensureChannelExists(channelId: string): Promise - getVideos(channel: YtChannel, ids: string[]): Promise -} - -export class YtDlpClient implements IOpenYTApi { - private ytdlpPath: string - private exec - - constructor() { - this.exec = promisify(exec) - this.ytdlpPath = `${pkgDir.sync(__dirname)}/node_modules/youtube-dl-exec/bin/yt-dlp` - } - - async ensureChannelExists(channelId: string): Promise { - // "-I :1" is used to only fetch at most one video of the channel if it exists. - return (await this.exec(`${this.ytdlpPath} --print channel_id https://www.youtube.com/channel/${channelId} -I :1`)) - .stdout - } - - async getVideos(channel: YtChannel, ids: string[]): Promise { - const videosMetadata: YtDlpVideoOutput[] = [] - const idsChunks = _.chunk(ids, 50) - - for (const idsChunk of idsChunks) { - const videosMetadataChunk = await Promise.all( - idsChunk.map(async (id) => { - const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`) - return JSON.parse(stdout) as YtDlpVideoOutput - }) - ) - videosMetadata.push(...videosMetadataChunk) - } - - return this.mapVideos(videosMetadata, channel) - } - - private mapVideos(videosMetadata: YtDlpVideoOutput[], channel: YtChannel): YtVideo[] { - return videosMetadata - .map( - (video) => - { - id: video?.id, - description: video?.description || '', - title: video?.title, - channelId: video?.channel_id, - thumbnails: { - high: `https://i.ytimg.com/vi/${video.id}/hqdefault.jpg`, - medium: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`, - standard: `https://i.ytimg.com/vi/${video.id}/sddefault.jpg`, - default: `https://i.ytimg.com/vi/${video.id}/default.jpg`, - }, - url: `https://youtube.com/watch?v=${video.id}`, - publishedAt: moment(video?.upload_date, 'YYYYMMDD').toDate().toISOString(), - createdAt: new Date(), - category: channel.videoCategoryId, - languageIso: channel.joystreamChannelLanguageIso, - privacyStatus: video?.availability || 'public', - liveBroadcastContent: - video?.live_status === 'is_upcoming' ? 'upcoming' : video?.live_status === 'is_live' ? 'live' : 'none', - license: - video?.license === 'Creative Commons Attribution license (reuse allowed)' ? 'creativeCommon' : 'youtube', - duration: video?.duration, - container: video?.ext, - viewCount: video?.view_count || 0, - state: 'New', - } - ) - .filter((v) => v.privacyStatus === 'public' && v.liveBroadcastContent === 'none') - } - - async getVideosIDs( - channel: YtChannel, - limit?: number, - order?: 'first' | 'last', - videoType: ('videos' | 'shorts' | 'streams')[] = ['videos', 'shorts'] // Excluding the live streams from syncing - ): Promise { - if (limit === undefined && order !== undefined) { - throw new Error('Order should only be provided if limit is provided') - } - - let limitOption = '' - if (limit) { - limitOption = !order || order === 'first' ? `-I :${limit}` : `-I -${limit}:-1` - } - - const allVideos = await Promise.all( - videoType.map(async (type) => { - try { - const { stdout } = await this.exec( - `${this.ytdlpPath} --extractor-args "youtubetab:approximate_date" -J --flat-playlist ${limitOption} https://www.youtube.com/channel/${channel.id}/${type}`, - { maxBuffer: Number.MAX_SAFE_INTEGER } - ) - - const videos: YtDlpFlatPlaylistOutput = [] - JSON.parse(stdout).entries.forEach((category: any) => { - if (category.entries) { - category.entries.forEach((video: any) => { - videos.push({ id: video.id, publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */ }) - }) - } else { - videos.push({ - id: category.id, - publishedAt: new Date(category.timestamp * 1000) /** Convert UNIX to date */, - }) - } - }) - - return videos - } catch (err) { - if (err instanceof Error && err.message.includes(`This channel does not have a ${type} tab`)) { - return [] - } - throw err - } - }) - ) - - // Flatten all videos and then sort based on the `order` parameter - const flattenedAndSortedVideos = allVideos.flat().sort((a, b) => { - if (order === 'last') { - return a.publishedAt.getTime() - b.publishedAt.getTime() // Oldest first - } else { - // Default to 'first' if order is not provided - return b.publishedAt.getTime() - a.publishedAt.getTime() // Most recent first - } - }) - return limit ? flattenedAndSortedVideos.slice(0, limit) : flattenedAndSortedVideos - } -} - -export interface IYoutubeApi { - ytdlpClient: YtDlpClient +interface IDataApiV3 { getUserFromCode(code: string, youtubeRedirectUri: string): Promise getChannel(user: Pick): Promise getVerifiedChannel( user: Pick ): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> getVideos(channel: YtChannel, ids: string[]): Promise - downloadVideo(videoUrl: string, outPath: string): ReturnType - getCreatorOnboardingRequirements(): ReadonlyConfig['creatorOnboardingRequirements'] } -export interface IQuotaMonitoringClient { +interface IQuotaMonitoringDataApiV3 extends IDataApiV3 { getQuotaUsage(): Promise getQuotaLimit(): Promise } -class YoutubeClient implements IYoutubeApi { +class DataApiV3 implements IDataApiV3 { private config: ReadonlyConfig readonly ytdlpClient: YtDlpClient constructor(config: ReadonlyConfig) { this.config = config - this.ytdlpClient = new YtDlpClient() - } - - getCreatorOnboardingRequirements() { - return this.config.creatorOnboardingRequirements + this.ytdlpClient = new YtDlpClient(config) } private getAuth(youtubeRedirectUri?: string) { - return new OAuth2Client({ - clientId: this.config.youtube.clientId, - clientSecret: this.config.youtube.clientSecret, - redirectUri: youtubeRedirectUri, - }) + if (this.config.youtube.apiMode !== 'api-free') { + return new OAuth2Client({ + clientId: this.config.youtube.api.clientId, + clientSecret: this.config.youtube.api.clientSecret, + redirectUri: youtubeRedirectUri, + }) + } else { + throw new Error('getAuth: Youtube API is not enabled') + } } - private getYoutube(accessToken: string, refreshToken: string) { + private getYoutube(accessToken?: string, refreshToken?: string) { const auth = this.getAuth() auth.setCredentials({ access_token: accessToken, @@ -239,13 +101,20 @@ class YoutubeClient implements IYoutubeApi { ) } - const user: YtUser = { + const channel = await this.getChannel({ id: tokenInfo.sub, + accessToken: tokenResponse.access_token, + refreshToken: tokenResponse.refresh_token, + }) + + const user: YtUser = { + id: channel.id, email: tokenInfo.email, accessToken: tokenResponse.access_token, refreshToken: tokenResponse.refresh_token, authorizationCode: code, joystreamMemberId: undefined, + youtubeVideoUrl: undefined, createdAt: new Date(), } return user @@ -374,18 +243,6 @@ class YoutubeClient implements IYoutubeApi { } } - async downloadVideo(videoUrl: string, outPath: string): ReturnType { - const response = await ytdl(videoUrl, { - noWarnings: true, - printJson: true, - format: 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/bv[height<=1080][ext=webm]+ba[ext=webm]/best[height<=1080]', - output: `${outPath}/%(id)s.%(ext)s`, - ffmpegLocation: ffmpegInstaller.path, - proxy: this.config.proxy?.url, - }) - return response - } - private async iterateVideos(youtube: youtube_v3.Youtube, channel: YtChannel, ids: string[]) { let videos: YtVideo[] = [] @@ -430,7 +287,6 @@ class YoutubeClient implements IYoutubeApi { id: channel.id, description: channel.snippet?.description, title: channel.snippet?.title, - userId: user.id, customUrl: channel.snippet?.customUrl, userAccessToken: user.accessToken, userRefreshToken: user.refreshToken, @@ -438,7 +294,6 @@ class YoutubeClient implements IYoutubeApi { default: channel.snippet?.thumbnails?.default?.url, medium: channel.snippet?.thumbnails?.medium?.url, high: channel.snippet?.thumbnails?.high?.url, - standard: channel.snippet?.thumbnails?.standard?.url, }, statistics: { viewCount: parseInt(channel.statistics?.viewCount ?? '0'), @@ -475,7 +330,6 @@ class YoutubeClient implements IYoutubeApi { thumbnails: { high: video.snippet?.thumbnails?.high?.url, medium: video.snippet?.thumbnails?.medium?.url, - standard: video.snippet?.thumbnails?.standard?.url, default: video.snippet?.thumbnails?.default?.url, }, url: `https://youtube.com/watch?v=${video.id}`, @@ -506,20 +360,27 @@ class YoutubeClient implements IYoutubeApi { } } -class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { +export class QuotaMonitoringDataApiV3 implements IQuotaMonitoringDataApiV3 { private quotaMonitoringClient: MetricServiceClient | undefined private googleCloudProjectId: string private DEFAULT_MAX_ALLOWED_QUOTA_USAGE = 95 // 95% + private dataApiV3: IDataApiV3 - constructor(private decorated: IYoutubeApi, private config: ReadonlyConfig, private statsRepo: StatsRepository) { - // Use the client id to get the google cloud project id - this.googleCloudProjectId = this.config.youtube.clientId.split('-')[0] + constructor(private config: ReadonlyConfig, private statsRepo: StatsRepository) { + this.dataApiV3 = new DataApiV3(config) - // if we have a key file path, we use it to authenticate with the google cloud monitoring api - if (this.config.youtube.adcKeyFilePath) { - this.quotaMonitoringClient = new MetricServiceClient({ - keyFile: path.resolve(this.config.youtube.adcKeyFilePath), - }) + if (this.config.youtube.apiMode !== 'api-free') { + // Use the client id to get the google cloud project id + this.googleCloudProjectId = this.config.youtube.api?.clientId.split('-')[0] + + // if we have a key file path, we use it to authenticate with the google cloud monitoring api + if (this.config.youtube.api.adcKeyFilePath) { + this.quotaMonitoringClient = new MetricServiceClient({ + keyFile: path.resolve(this.config.youtube.api?.adcKeyFilePath), + }) + } + } else { + throw new Error(`QuotaMonitoringDataApiV3: Youtube API is not enabled`) } } @@ -597,25 +458,13 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { return quotaUsage } - /** - * Implement IYoutubeApi interface - **/ - - get ytdlpClient() { - return this.decorated.ytdlpClient - } - - getCreatorOnboardingRequirements() { - return this.decorated.getCreatorOnboardingRequirements() - } - getUserFromCode(code: string, youtubeRedirectUri: string) { - return this.decorated.getUserFromCode(code, youtubeRedirectUri) + return this.dataApiV3.getUserFromCode(code, youtubeRedirectUri) } async getVerifiedChannel(user: Pick) { // These is no api quota check for this operation, as we allow untracked access to channel verification/signup endpoint. - const verifiedChannel = await this.decorated.getVerifiedChannel(user) + const verifiedChannel = await this.dataApiV3.getVerifiedChannel(user) // increase used quota count by 1 because only one page is returned await this.increaseUsedQuota({ signupQuotaIncrement: 1 }) @@ -633,7 +482,7 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { } // get channels from api - const channels = await this.decorated.getChannel(user) + const channels = await this.dataApiV3.getChannel(user) // increase used quota count by 1 because only one page is returned await this.increaseUsedQuota({ syncQuotaIncrement: 1 }) @@ -651,7 +500,7 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { } // get videos from api - const videos = await this.decorated.getVideos(channel, ids) + const videos = await this.dataApiV3.getVideos(channel, ids) // increase used quota count, 1 api call is being used per page of 50 videos await this.increaseUsedQuota({ syncQuotaIncrement: Math.ceil(videos.length / 50) }) @@ -659,10 +508,6 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { return videos } - downloadVideo(videoUrl: string, outPath: string): ReturnType { - return this.decorated.downloadVideo(videoUrl, outPath) - } - private async increaseUsedQuota({ syncQuotaIncrement = 0, signupQuotaIncrement = 0 }) { // Quota resets at Pacific Time, and pst is 8 hours behind UTC const stats = await this.statsRepo.getOrSetTodaysStats() @@ -675,20 +520,14 @@ class QuotaMonitoringClient implements IQuotaMonitoringClient, IYoutubeApi { } private async canCallYoutube(): Promise { - if (this.quotaMonitoringClient) { + if (this.config.youtube.apiMode !== 'api-free' && this.quotaMonitoringClient) { const quotaUsage = await this.getQuotaUsage() const quotaLimit = await this.getQuotaLimit() return ( (quotaUsage * 100) / quotaLimit < - (this.config.youtube.maxAllowedQuotaUsageInPercentage || this.DEFAULT_MAX_ALLOWED_QUOTA_USAGE) + (this.config.youtube.api?.maxAllowedQuotaUsageInPercentage || this.DEFAULT_MAX_ALLOWED_QUOTA_USAGE) ) } return true } } - -export const YoutubeApi = { - create(config: ReadonlyConfig, statsRepo: StatsRepository): IYoutubeApi { - return new QuotaMonitoringClient(new YoutubeClient(config), config, statsRepo) - }, -} diff --git a/src/services/youtube/index.ts b/src/services/youtube/index.ts new file mode 100644 index 00000000..2c079733 --- /dev/null +++ b/src/services/youtube/index.ts @@ -0,0 +1,25 @@ +import { StatsRepository } from '../../repository' +import { ReadonlyConfig } from '../../types' +import { QuotaMonitoringDataApiV3 } from './api' +import { YtDlpClient } from './openApi' +import { YoutubeOperationalApi } from './operationalApi' + +export class YoutubeApi { + public readonly dataApiV3: QuotaMonitoringDataApiV3 + public readonly ytdlp: YtDlpClient + public readonly operationalApi: YoutubeOperationalApi + + constructor(private config: ReadonlyConfig, statsRepo: StatsRepository) { + this.ytdlp = new YtDlpClient(config) + + this.operationalApi = new YoutubeOperationalApi(config) + + if (this.config.youtube.apiMode !== 'api-free') { + this.dataApiV3 = new QuotaMonitoringDataApiV3(config, statsRepo) + } + } + + getCreatorOnboardingRequirements() { + return this.config.creatorOnboardingRequirements + } +} diff --git a/src/services/youtube/openApi.ts b/src/services/youtube/openApi.ts new file mode 100644 index 00000000..fe48fcce --- /dev/null +++ b/src/services/youtube/openApi.ts @@ -0,0 +1,179 @@ +import ffmpegInstaller from '@ffmpeg-installer/ffmpeg' +import { exec } from 'child_process' +import _ from 'lodash' +import moment from 'moment-timezone' +import pkgDir from 'pkg-dir' +import { promisify } from 'util' +import ytdl from 'youtube-dl-exec' +import { ReadonlyConfig } from '../../types' +import { YtChannel, YtDlpFlatPlaylistOutput, YtDlpVideoOutput, YtUser, YtVideo } from '../../types/youtube' + +export interface IOpenYTApi { + getChannel(channelId: string): Promise<{ id: string; title: string; description: string }> + getVideoFromUrl(videoUrl: string): Promise + getVideos(channel: YtChannel, ids: string[]): Promise + downloadVideo(videoUrl: string, outPath: string): ReturnType + getUserFromVideoUrl(videoUrl: string): Promise +} + +export class YtDlpClient implements IOpenYTApi { + private ytdlpPath: string + private exec + + constructor(private config: ReadonlyConfig) { + this.exec = promisify(exec) + this.ytdlpPath = `${pkgDir.sync(__dirname)}/node_modules/youtube-dl-exec/bin/yt-dlp` + } + + async getChannel(channelId: string): Promise<{ id: string; title: string; description: string }> { + // "-I :0" is used to not fetch any of the videos of the channel but only it's metadata + const output = ( + await this.exec(`${this.ytdlpPath} --skip-download -J https://www.youtube.com/channel/${channelId} -I :0`) + ).stdout + + return JSON.parse(output) as { id: string; title: string; description: string } + } + + async getVideoFromUrl(videoUrl: string): Promise { + const { stdout } = await this.exec(`${this.ytdlpPath} -J ${videoUrl}`) + const output = JSON.parse(stdout) as YtDlpVideoOutput + return this.mapVideo(output) + } + + async getUserFromVideoUrl(videoUrl: string): Promise { + const { stdout } = await this.exec(`${this.ytdlpPath} --print channel_id ${videoUrl}`) + + const user: YtUser = { + id: stdout.replace(/(\r\n|\n|\r)/gm, ''), + email: undefined, + accessToken: undefined, + refreshToken: undefined, + authorizationCode: undefined, + joystreamMemberId: undefined, + youtubeVideoUrl: videoUrl, + createdAt: new Date(), + } + return user + } + + async downloadVideo(videoUrl: string, outPath: string): ReturnType { + const response = await ytdl(videoUrl, { + noWarnings: true, + printJson: true, + format: 'bv[height<=1080][ext=mp4]+ba[ext=m4a]/bv[height<=1080][ext=webm]+ba[ext=webm]/best[height<=1080]', + output: `${outPath}/%(id)s.%(ext)s`, + ffmpegLocation: ffmpegInstaller.path, + proxy: this.config.proxy?.url, + }) + return response + } + + async getVideos(channel: YtChannel, ids: string[]): Promise { + const videosMetadata: YtDlpVideoOutput[] = [] + const idsChunks = _.chunk(ids, 50) + + for (const idsChunk of idsChunks) { + const videosMetadataChunk = await Promise.all( + idsChunk.map(async (id) => { + const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`) + return JSON.parse(stdout) as YtDlpVideoOutput + }) + ) + videosMetadata.push(...videosMetadataChunk) + } + + return this.mapVideos(videosMetadata, channel) + } + + private mapVideo(video: YtDlpVideoOutput, channel?: YtChannel): YtVideo { + return { + id: video?.id, + description: video?.description || '', + title: video?.title, + channelId: video?.channel_id, + thumbnails: { + high: `https://i.ytimg.com/vi/${video.id}/hqdefault.jpg`, + medium: `https://i.ytimg.com/vi/${video.id}/mqdefault.jpg`, + default: `https://i.ytimg.com/vi/${video.id}/default.jpg`, + }, + url: `https://youtube.com/watch?v=${video.id}`, + publishedAt: moment(video?.upload_date, 'YYYYMMDD').toDate().toISOString(), + createdAt: new Date(), + category: channel?.videoCategoryId, + languageIso: channel?.joystreamChannelLanguageIso, + privacyStatus: video?.availability || 'public', + liveBroadcastContent: + video?.live_status === 'is_upcoming' ? 'upcoming' : video?.live_status === 'is_live' ? 'live' : 'none', + license: video?.license === 'Creative Commons Attribution license (reuse allowed)' ? 'creativeCommon' : 'youtube', + duration: video?.duration, + container: video?.ext, + viewCount: video?.view_count || 0, + state: 'New', + } + } + + private mapVideos(videosMetadata: YtDlpVideoOutput[], channel: YtChannel): YtVideo[] { + return videosMetadata + .map((video) => this.mapVideo(video, channel)) + .filter((v) => v.privacyStatus === 'public' && v.liveBroadcastContent === 'none') + } + + async getVideosIDs( + channel: YtChannel, + limit?: number, + order?: 'first' | 'last', + videoType: ('videos' | 'shorts' | 'streams')[] = ['videos', 'shorts'] // Excluding the live streams from syncing + ): Promise { + if (limit === undefined && order !== undefined) { + throw new Error('Order should only be provided if limit is provided') + } + + let limitOption = '' + if (limit) { + limitOption = !order || order === 'first' ? `-I :${limit}` : `-I -${limit}:-1` + } + + const allVideos = await Promise.all( + videoType.map(async (type) => { + try { + const { stdout } = await this.exec( + `${this.ytdlpPath} --extractor-args "youtubetab:approximate_date" -J --flat-playlist ${limitOption} https://www.youtube.com/channel/${channel.id}/${type}`, + { maxBuffer: Number.MAX_SAFE_INTEGER } + ) + + const videos: YtDlpFlatPlaylistOutput = [] + JSON.parse(stdout).entries.forEach((category: any) => { + if (category.entries) { + category.entries.forEach((video: any) => { + videos.push({ id: video.id, publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */ }) + }) + } else { + videos.push({ + id: category.id, + publishedAt: new Date(category.timestamp * 1000) /** Convert UNIX to date */, + }) + } + }) + + return videos + } catch (err) { + if (err instanceof Error && err.message.includes(`This channel does not have a ${type} tab`)) { + return [] + } + throw err + } + }) + ) + + // Flatten all videos and then sort based on the `order` parameter + const flattenedAndSortedVideos = allVideos.flat().sort((a, b) => { + if (order === 'last') { + return a.publishedAt.getTime() - b.publishedAt.getTime() // Oldest first + } else { + // Default to 'first' if order is not provided + return b.publishedAt.getTime() - a.publishedAt.getTime() // Most recent first + } + }) + return limit ? flattenedAndSortedVideos.slice(0, limit) : flattenedAndSortedVideos + } +} diff --git a/src/services/youtube/operationalApi.ts b/src/services/youtube/operationalApi.ts new file mode 100644 index 00000000..371aafb5 --- /dev/null +++ b/src/services/youtube/operationalApi.ts @@ -0,0 +1,211 @@ +import axios from 'axios' +import { ReadonlyConfig } from '../../types' +import { ExitCodes, YoutubeApiError } from '../../types/errors' +import { YtChannel, YtUser } from '../../types/youtube' +import { YtDlpClient } from './openApi' + +type OperationalApiChannelListResponse = { + kind: string + etag: string + items: Channel[] +} + +type Channel = { + kind: string + etag: string + id: string + status: any + countryChannelId: string + about: { + stats: { + joinedDate: number + viewCount: number + subscriberCount: number + videoCount: number + } + description: string + details: { + location: string + } + handle: string + } + snippet: { + avatar: Image[] + banner: Image[] + } +} + +type Image = { + url: string +} + +const YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP = `I want to be in YPP` + +export class YoutubeOperationalApi { + private baseUrl: string + readonly ytdlpClient: YtDlpClient + + constructor(private config: ReadonlyConfig) { + this.ytdlpClient = new YtDlpClient(config) + this.baseUrl = config.youtube.operationalApi.url + } + + async getChannel(channelId: string): Promise { + try { + const [ytdlpApiResponse, operationalApiResponse] = await Promise.all([ + this.ytdlpClient.getChannel(channelId), + axios.get(`${this.baseUrl}/channels?part=status,about,snippet&id=${channelId}`), + ]) + + return this.mapChannel( + (operationalApiResponse.data as OperationalApiChannelListResponse).items[0], + ytdlpApiResponse + ) + } catch (error) { + throw error + } + } + + async getVerifiedChannel(user: YtUser): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> { + const { + minimumSubscribersCount, + minimumVideosCount, + minimumChannelAgeHours, + minimumVideoAgeHours, + minimumVideosPerMonth, + monthsToConsider, + } = this.config.creatorOnboardingRequirements + + const errors: YoutubeApiError[] = [] + + const video = await this.ytdlpClient.getVideoFromUrl(user.youtubeVideoUrl!) + + if (video.privacyStatus !== 'unlisted') { + errors.push( + new YoutubeApiError( + ExitCodes.YoutubeApi.VIDEO_PRIVACY_STATUS_NOT_UNLISTED, + `Video ${video.id} is not unlisted`, + video.privacyStatus, + 'unlisted' + ) + ) + } + + if (video.title !== YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP) { + errors.push( + new YoutubeApiError( + ExitCodes.YoutubeApi.VIDEO_TITLE_NOT_MATCHING, + `Video ${video.id} title is not matching`, + video.title, + YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP + ) + ) + } + + const channel = await this.getChannel(video.channelId) + if (channel.statistics.subscriberCount < minimumSubscribersCount) { + errors.push( + new YoutubeApiError( + ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_SUBSCRIBERS, + `Channel ${channel.id} with ${channel.statistics.subscriberCount} subscribers does not ` + + `meet Youtube Partner Program requirement of ${minimumSubscribersCount} subscribers`, + channel.statistics.subscriberCount, + minimumSubscribersCount + ) + ) + } + + // at least 'minimumVideosCount' videos should be 'minimumVideoAgeHours' old + const videoCreationTimeCutoff = new Date() + videoCreationTimeCutoff.setHours(videoCreationTimeCutoff.getHours() - minimumVideoAgeHours) + + // filter all videos that are older than 'minimumVideoAgeHours' + const oldVideos = (await this.ytdlpClient.getVideosIDs(channel, minimumVideosCount, 'last')).filter( + (v) => v.publishedAt < videoCreationTimeCutoff + ) + if (oldVideos.length < minimumVideosCount) { + errors.push( + new YoutubeApiError( + ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_VIDEOS, + `Channel ${channel.id} with ${oldVideos.length} videos does not meet Youtube ` + + `Partner Program requirement of at least ${minimumVideosCount} videos, each ${( + minimumVideoAgeHours / 720 + ).toPrecision(2)} month old`, + channel.statistics.videoCount, + minimumVideosCount + ) + ) + } + + // TODO: make configurable (currently hardcoded to latest 1 month) + // at least 'minimumVideosPerMonth' should be there for 'monthsToConsider' + const nMonthsAgo = new Date() + nMonthsAgo.setMonth(nMonthsAgo.getMonth() - 1) + + const newVideos = (await this.ytdlpClient.getVideosIDs(channel, minimumVideosPerMonth)).filter( + (v) => v.publishedAt > nMonthsAgo + ) + if (newVideos.length < minimumVideosPerMonth) { + errors.push( + new YoutubeApiError( + ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_NEW_VIDEOS_REQUIREMENT, + `Channel ${channel.id} videos does not meet Youtube Partner Program ` + + `requirement of at least ${minimumVideosPerMonth} video per ` + + `month, posted over the last ${monthsToConsider} months`, + channel.statistics.videoCount, + minimumVideosPerMonth + ) + ) + } + + // Channel should be at least 'minimumChannelAgeHours' old + const channelCreationTimeCutoff = new Date() + channelCreationTimeCutoff.setHours(channelCreationTimeCutoff.getHours() - minimumChannelAgeHours) + if (new Date(channel.publishedAt) > channelCreationTimeCutoff) { + errors.push( + new YoutubeApiError( + ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_CREATION_DATE, + `Channel ${channel.id} with creation time of ${channel.publishedAt} does not ` + + `meet Youtube Partner Program requirement of channel being at least ${( + minimumChannelAgeHours / 720 + ).toPrecision(2)} months old`, + channel.publishedAt, + channelCreationTimeCutoff + ) + ) + } + + return { channel, errors } + } + + private mapChannel(channel: Channel, ytdlpApiResponse: { title: string }): YtChannel { + return { + id: channel.id, + description: channel.about.description, + title: ytdlpApiResponse.title, + customUrl: channel.about.handle, + thumbnails: { + default: channel.snippet?.avatar[0].url, + medium: channel.snippet?.avatar[1].url, + high: channel.snippet?.avatar[2].url, + }, + statistics: { + viewCount: channel.about.stats.viewCount, + subscriberCount: channel.about.stats.subscriberCount, + videoCount: channel.about.stats.videoCount, + commentCount: 0, + }, + historicalVideoSyncedSize: 0, + bannerImageUrl: channel.snippet.banner[0]?.url, + // language: channel.snippet?.defaultLanguage, // TODO: find workaround for channel + publishedAt: new Date(channel.about.stats?.joinedDate).toISOString(), + performUnauthorizedSync: false, + shouldBeIngested: true, + allowOperatorIngestion: true, + yppStatus: 'Unverified', + createdAt: new Date(), + lastActedAt: new Date(), + phantomKey: 'phantomData', + } + } +} From 169a0fb3bc368680273bc253f2b854ce9f861262 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 12 Jan 2024 17:46:27 +0500 Subject: [PATCH 04/17] update type definitions --- src/types/errors.ts | 2 ++ src/types/generated/ConfigJson.d.ts | 23 ++++++++++++++++++--- src/types/index.ts | 14 +++++++++++-- src/types/youtube.ts | 32 ++++++++++++++++------------- 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/src/types/errors.ts b/src/types/errors.ts index 618a4562..06882082 100644 --- a/src/types/errors.ts +++ b/src/types/errors.ts @@ -2,6 +2,8 @@ export namespace ExitCodes { export enum YoutubeApi { CHANNEL_NOT_FOUND = 'CHANNEL_NOT_FOUND', VIDEO_NOT_FOUND = 'VIDEO_NOT_FOUND', + VIDEO_PRIVACY_STATUS_NOT_UNLISTED = 'VIDEO_PRIVACY_STATUS_NOT_UNLISTED', // TODO: Add this to the error message + VIDEO_TITLE_NOT_MATCHING = 'VIDEO_TITLE_NOT_MATCHING', // TODO: Add this to the error message CHANNEL_ALREADY_REGISTERED = 'CHANNEL_ALREADY_REGISTERED', CHANNEL_STATUS_SUSPENDED = 'CHANNEL_STATUS_SUSPENDED', CHANNEL_CRITERIA_UNMET_SUBSCRIBERS = 'CHANNEL_CRITERIA_UNMET_SUBSCRIBERS', diff --git a/src/types/generated/ConfigJson.d.ts b/src/types/generated/ConfigJson.d.ts index 4013fbeb..3ae061d8 100644 --- a/src/types/generated/ConfigJson.d.ts +++ b/src/types/generated/ConfigJson.d.ts @@ -69,7 +69,7 @@ export interface YoutubeSyncNodeConfiguration { console?: ConsoleLoggingOptions elastic?: ElasticsearchLoggingOptions } - youtube: YoutubeOauth2ClientConfiguration + youtube: YoutubeRelatedConfiguration aws?: AWSConfigurationsNeededToConnectWithDynamoDBInstance proxy?: Socks5ProxyClientConfigurationUsedByYtDlpToBypassIPBlockageByYoutube /** @@ -184,9 +184,17 @@ export interface ElasticsearchAuthenticationOptions { password: string } /** - * Youtube Oauth2 Client configuration + * Youtube related configuration */ -export interface YoutubeOauth2ClientConfiguration { +export interface YoutubeRelatedConfiguration { + apiMode: 'api-free' | 'api' | 'both' + api?: YoutubeAPIConfiguration + operationalApi: YoutubeOperationalAPIHttpsGithubComBenjaminLoisonYouTubeOperationalAPIConfiguration +} +/** + * Youtube API configuration + */ +export interface YoutubeAPIConfiguration { /** * Youtube Oauth2 Client Id */ @@ -204,6 +212,15 @@ export interface YoutubeOauth2ClientConfiguration { */ adcKeyFilePath?: string } +/** + * Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration + */ +export interface YoutubeOperationalAPIHttpsGithubComBenjaminLoisonYouTubeOperationalAPIConfiguration { + /** + * URL of the Youtube Operational API server (for example: http://localhost:8080) + */ + url: string +} /** * AWS configurations needed to connect with DynamoDB instance */ diff --git a/src/types/index.ts b/src/types/index.ts index e3080a63..2c2c01eb 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -14,9 +14,19 @@ type SyncDisabled = Omit, 'lim limits?: Omit, 'storage'> & { storage: number } } -export type Config = Omit & { +type YoutubeApiEnabled = Omit & { + apiMode: 'api' | 'both' + api: NonNullable +} + +type YoutubeApiDisabled = Omit & { + apiMode: 'api-free' +} + +export type Config = Omit & { version: string sync: SyncEnabled | SyncDisabled + youtube: YoutubeApiEnabled | YoutubeApiDisabled } export type ReadonlyConfig = DeepReadonly @@ -28,7 +38,7 @@ export type DisplaySafeConfig = Omit youtube: Secret joystream: Secret - logs?: { elastic: Secret['elastic']> } + logs?: { elastic: Secret['elastic']> } aws?: { credentials: Secret['credentials']> } } diff --git a/src/types/youtube.ts b/src/types/youtube.ts index ed8b47bc..bd07bf4d 100644 --- a/src/types/youtube.ts +++ b/src/types/youtube.ts @@ -3,6 +3,7 @@ import { VideoMetadataAndHash } from '../services/syncProcessing/ContentMetadata type DeploymentEnv = 'dev' | 'local' | 'testing' | 'prod' const deploymentEnv = process.env.DEPLOYMENT_ENV as DeploymentEnv | undefined +// TODO: only allow sharing unlisted videos (because if we allow sharing public videos, then anyone can share video before the creator) export type ResourcePrefix = `${Exclude}_` | '' export const resourcePrefix = (deploymentEnv && deploymentEnv !== 'prod' ? `${deploymentEnv}_` : '') as ResourcePrefix @@ -10,9 +11,6 @@ export class YtChannel { // Channel ID id: string - // ID of the user that owns the channel - userId: string - // Youtube channel custom URL. Also known as youtube channel handle customUrl: string @@ -167,21 +165,29 @@ export class YtChannel { } } +/** + * We use the same DynamoDB table for storing users/channels verified through + * Oauth api vs non-api (i.e. through shared a URL for a required video). + */ export class YtUser { - // Youtube user ID + // Youtube channel ID id: string - // Youtube User email - email: string + // Youtube User/Channel email (will only be available for users signed up through api workflow) + email: string | undefined + + // User access token (will only be available for users signed up through api workflow) + accessToken: string | undefined - // User access token - accessToken: string + // User refresh token (will only be available for users signed up through api workflow) + refreshToken: string | undefined - // User refresh token - refreshToken: string + // User authorization code (will only be available for users signed up through api workflow) + authorizationCode: string | undefined - // User authorization code - authorizationCode: string + // The URL for a specific video of Youtube channel with which the user is trying to register + // for YPP program (will only be available for users signed up through api-free workflow) + youtubeVideoUrl: string | undefined // Corresponding Joystream member ID for Youtube user joystreamMemberId: number | undefined @@ -194,7 +200,6 @@ export type Thumbnails = { default: string medium: string high: string - standard: string } export enum VideoUnavailableReasons { @@ -337,7 +342,6 @@ export const getImages = (channel: YtChannel) => { ...urlAsArray(channel.thumbnails.default), ...urlAsArray(channel.thumbnails.high), ...urlAsArray(channel.thumbnails.medium), - ...urlAsArray(channel.thumbnails.standard), ] } From 00ee8b043aede0eb65c0bd6dc4813ac0f11e938c Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 12 Jan 2024 17:49:41 +0500 Subject: [PATCH 05/17] update channels db table schema --- src/infrastructure/index.ts | 7 +------ src/repository/channel.ts | 42 +++++++------------------------------ src/repository/user.ts | 3 +++ 3 files changed, 12 insertions(+), 40 deletions(-) diff --git a/src/infrastructure/index.ts b/src/infrastructure/index.ts index 57885d8a..d89fbdeb 100644 --- a/src/infrastructure/index.ts +++ b/src/infrastructure/index.ts @@ -17,13 +17,8 @@ const userTable = new aws.dynamodb.Table('users', { const channelsTable = new aws.dynamodb.Table('channels', { name: `${resourcePrefix}channels`, - hashKey: nameof('userId'), - rangeKey: nameof('id'), + hashKey: nameof('id'), attributes: [ - { - name: nameof('userId'), - type: 'S', - }, { name: nameof('id'), type: 'S', diff --git a/src/repository/channel.ts b/src/repository/channel.ts index af2f4aa7..c55a75f7 100644 --- a/src/repository/channel.ts +++ b/src/repository/channel.ts @@ -20,12 +20,6 @@ function createChannelModel(tablePrefix: ResourcePrefix) { { // ID of the Youtube channel id: { - type: String, - rangeKey: true, - }, - - // ID of the user that owns the channel - userId: { type: String, hashKey: true, }, @@ -255,8 +249,8 @@ export class ChannelsRepository implements IRepository { async save(channel: YtChannel): Promise { return this.withLock(async () => { - const update = omit(['id', 'userId', 'updatedAt'], channel) - const result = await this.model.update({ id: channel.id, userId: channel.userId }, update) + const update = omit(['id', 'updatedAt'], channel) + const result = await this.model.update({ id: channel.id }, update) return mapTo(result) }) } @@ -268,16 +262,16 @@ export class ChannelsRepository implements IRepository { return this.withLock(async () => { const updateTransactions = channels.map((channel) => { - const update = omit(['id', 'userId', 'updatedAt'], channel) - return this.model.transaction.update({ id: channel.id, userId: channel.userId }, update) + const update = omit(['id', 'updatedAt'], channel) + return this.model.transaction.update({ id: channel.id }, update) }) return dynamoose.transaction(updateTransactions) }) } - async delete(id: string, userId: string): Promise { + async delete(id: string): Promise { return this.withLock(async () => { - await this.model.delete({ id, userId }) + await this.model.delete({ id }) return }) } @@ -325,7 +319,7 @@ export class ChannelsService { q .sort('descending') .filter('yppStatus') - .eq('Verified') + .beginsWith('Verified::') .or() .filter('yppStatus') .eq('Unverified') @@ -350,33 +344,13 @@ export class ChannelsService { * @returns Returns channel by youtube channelId */ async getById(channelId: string): Promise { - const result = await this.channelsRepository.get(channelId.toString()) + const result = await this.channelsRepository.get(channelId) if (!result) { throw new Error(`Could not find channel with id ${channelId}`) } return result } - /** - * @param userId - * @returns Returns channel by userId - */ - async getByUserId(userId: string): Promise { - const [result] = await this.channelsRepository.query('userId', (q) => q.eq(userId)) - if (!result) { - throw new Error(`Could not find user with id ${userId}`) - } - return result - } - - /** - * @param userId - * @returns List of Channels for given user - */ - async getAll(userId: string): Promise { - return await this.channelsRepository.query({ userId }, (q) => q) - } - /** * @param count Number of record to retrieve * @returns List of `n` recent verified channels diff --git a/src/repository/user.ts b/src/repository/user.ts index d32dedb5..df27ddfc 100644 --- a/src/repository/user.ts +++ b/src/repository/user.ts @@ -27,6 +27,9 @@ function createUserModel(tablePrefix: ResourcePrefix) { // user authorization code authorizationCode: String, + // The URL for a specific video of Youtube channel with which the user is trying to register for YPP program + youtubeVideoUrl: String, + // user access token obtained from authorization code after successful authentication accessToken: String, From df6424cf5677d3050a5ebad4155b02b8d34eee35 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 12 Jan 2024 17:52:21 +0500 Subject: [PATCH 06/17] update http API --- src/app/index.ts | 6 +- .../sync/addUnauthorizedChannelForSyncing.ts | 1 - src/services/httpApi/api-spec.json | 63 ++++++-------- src/services/httpApi/controllers/channels.ts | 53 ++++++------ .../httpApi/controllers/membership.ts | 18 ++-- src/services/httpApi/controllers/status.ts | 2 +- src/services/httpApi/controllers/users.ts | 28 ++++--- src/services/httpApi/controllers/videos.ts | 6 +- src/services/httpApi/dtos.ts | 82 +++++++++++-------- src/services/httpApi/main.ts | 6 +- src/services/httpApi/utils.ts | 22 +++++ .../syncProcessing/ContentDownloadService.ts | 6 +- .../syncProcessing/YoutubePollingService.ts | 12 +-- src/services/syncProcessing/index.ts | 4 +- 14 files changed, 173 insertions(+), 136 deletions(-) create mode 100644 src/services/httpApi/utils.ts diff --git a/src/app/index.ts b/src/app/index.ts index cfef33ea..9623d108 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -10,14 +10,14 @@ import { RuntimeApi } from '../services/runtime/api' import { JoystreamClient } from '../services/runtime/client' import { ContentProcessingService } from '../services/syncProcessing' import { YoutubePollingService } from '../services/syncProcessing/YoutubePollingService' -import { IYoutubeApi, YoutubeApi } from '../services/youtube/api' +import { YoutubeApi } from '../services/youtube' import { Config, DisplaySafeConfig } from '../types' export class Service { private config: Config private logging: LoggingService private logger: Logger - private youtubeApi: IYoutubeApi + private youtubeApi: YoutubeApi private queryNodeApi: QueryNodeApi private dynamodbService: DynamodbService private runtimeApi: RuntimeApi @@ -32,7 +32,7 @@ export class Service { this.logger = this.logging.createLogger('Server') this.queryNodeApi = new QueryNodeApi(config.endpoints.queryNode, this.logging) this.dynamodbService = new DynamodbService(this.config.aws) - this.youtubeApi = YoutubeApi.create(this.config, this.dynamodbService.repo.stats) + this.youtubeApi = new YoutubeApi(this.config, this.dynamodbService.repo.stats) this.runtimeApi = new RuntimeApi(config.endpoints.joystreamNodeWs, this.logging) this.joystreamClient = new JoystreamClient(config, this.runtimeApi, this.queryNodeApi, this.logging) diff --git a/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts b/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts index 2e955eeb..c2ade650 100644 --- a/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts +++ b/src/cli/commands/sync/addUnauthorizedChannelForSyncing.ts @@ -172,7 +172,6 @@ export default class AddUnauthorizedChannelForSyncing extends RuntimeApiCommandB const c = await dynamo.channels.save({ id: ytChannel.author.channelID, title: ytChannel.author.name, - userId: `UnauthorizedUser-${ytChannel.author.channelID}`, joystreamChannelId, createdAt: new Date(), lastActedAt: new Date(), diff --git a/src/services/httpApi/api-spec.json b/src/services/httpApi/api-spec.json index 5b5742dd..a1708d00 100644 --- a/src/services/httpApi/api-spec.json +++ b/src/services/httpApi/api-spec.json @@ -1,14 +1,14 @@ { "openapi": "3.0.0", "paths": { - "/users/{userId}/videos": { + "/channels/{channelId}/videos": { "get": { "operationId": "VideosController_get", "summary": "", "description": "Get videos across all channels owned by the user", "parameters": [ { - "name": "userId", + "name": "channelId", "required": true, "in": "path", "schema": { @@ -752,7 +752,7 @@ "post": { "operationId": "MembershipController_createMembership", "summary": "", - "description": "Create Joystream's on-chain Membership for a verfifed YPP user. It will forward request\n to Joystream faucet with an Authorization header to circumvent captcha verfication by the faucet", + "description": "Create Joystream's on-chain Membership for a verified YPP user. It will forward request\n to Joystream faucet with an Authorization header to circumvent captcha verification by the faucet", "parameters": [], "requestBody": { "required": true, @@ -802,10 +802,10 @@ "authorizationCode": { "type": "string" }, - "userId": { + "youtubeVideoUrl": { "type": "string" }, - "email": { + "id": { "type": "string" }, "joystreamChannelId": { @@ -823,28 +823,13 @@ }, "required": [ "authorizationCode", - "userId", - "email", + "youtubeVideoUrl", + "id", "joystreamChannelId", "shouldBeIngested", "videoCategoryId" ] }, - "UserDto": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "email": { - "type": "string" - } - }, - "required": [ - "id", - "email" - ] - }, "ThumbnailsDto": { "type": "object", "properties": { @@ -856,16 +841,12 @@ }, "high": { "type": "string" - }, - "standard": { - "type": "string" } }, "required": [ "default", "medium", - "high", - "standard" + "high" ] }, "ChannelDto": { @@ -943,15 +924,11 @@ "SaveChannelResponse": { "type": "object", "properties": { - "user": { - "$ref": "#/components/schemas/UserDto" - }, "channel": { "$ref": "#/components/schemas/ChannelDto" } }, "required": [ - "user", "channel" ] }, @@ -1129,6 +1106,8 @@ "enum": [ "CHANNEL_NOT_FOUND", "VIDEO_NOT_FOUND", + "VIDEO_PRIVACY_STATUS_NOT_UNLISTED", + "VIDEO_TITLE_NOT_MATCHING", "CHANNEL_ALREADY_REGISTERED", "CHANNEL_STATUS_SUSPENDED", "CHANNEL_CRITERIA_UNMET_SUBSCRIBERS", @@ -1192,11 +1171,15 @@ }, "youtubeRedirectUri": { "type": "string" + }, + "youtubeVideoUrl": { + "type": "string" } }, "required": [ "authorizationCode", - "youtubeRedirectUri" + "youtubeRedirectUri", + "youtubeVideoUrl" ] }, "VerifyChannelResponse": { @@ -1205,7 +1188,7 @@ "email": { "type": "string" }, - "userId": { + "id": { "type": "string" }, "channelHandle": { @@ -1229,7 +1212,7 @@ }, "required": [ "email", - "userId", + "id", "channelHandle", "channelTitle", "channelDescription", @@ -1251,6 +1234,9 @@ "syncStatus": { "type": "string" }, + "apiMode": { + "type": "object" + }, "syncBacklog": { "type": "number" } @@ -1258,18 +1244,22 @@ "required": [ "version", "syncStatus", + "apiMode", "syncBacklog" ] }, "CreateMembershipRequest": { "type": "object", "properties": { - "userId": { + "id": { "type": "string" }, "authorizationCode": { "type": "string" }, + "youtubeVideoUrl": { + "type": "string" + }, "account": { "type": "string" }, @@ -1287,8 +1277,9 @@ } }, "required": [ - "userId", + "id", "authorizationCode", + "youtubeVideoUrl", "account", "handle", "avatar", diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts index a7258f83..eaff4cf6 100644 --- a/src/services/httpApi/controllers/channels.ts +++ b/src/services/httpApi/controllers/channels.ts @@ -16,14 +16,13 @@ import { } from '@nestjs/common' import { ApiBody, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' import { signatureVerify } from '@polkadot/util-crypto' -import { randomBytes } from 'crypto' import { DynamodbService } from '../../../repository' import { ReadonlyConfig } from '../../../types' -import { YtChannel, YtUser } from '../../../types/youtube' +import { YtChannel } from '../../../types/youtube' import { QueryNodeApi } from '../../query-node/api' import { ContentProcessingService } from '../../syncProcessing' import { YoutubePollingService } from '../../syncProcessing/YoutubePollingService' -import { IYoutubeApi } from '../../youtube/api' +import { YoutubeApi } from '../../youtube' import { ChannelDto, ChannelInductionRequirementsDto, @@ -36,7 +35,6 @@ import { SetOperatorIngestionStatusDto, SuspendChannelDto, UpdateChannelCategoryDto, - UserDto, VerifyChannelDto, WhitelistChannelDto, } from '../dtos' @@ -46,7 +44,7 @@ import { export class ChannelsController { constructor( @Inject('config') private config: ReadonlyConfig, - @Inject('youtube') private youtubeApi: IYoutubeApi, + private youtubeApi: YoutubeApi, private qnApi: QueryNodeApi, private dynamodbService: DynamodbService, private youtubePollingService: YoutubePollingService, @@ -57,18 +55,24 @@ export class ChannelsController { @ApiBody({ type: SaveChannelRequest }) @ApiResponse({ type: SaveChannelResponse }) @ApiOperation({ description: `Saves channel record of a YPP verified user` }) + // TODO: if channel exists dont allow to save again, async saveChannel(@Body() channelInfo: SaveChannelRequest): Promise { try { const { - userId, + id, authorizationCode, - email, + youtubeVideoUrl, joystreamChannelId, shouldBeIngested, videoCategoryId, referrerChannelId, } = channelInfo + // Ensure that channel is not already registered for YPP program + if (await this.dynamodbService.repo.channels.get(id)) { + throw new Error('Channel already exists, cannot re-save the channel') + } + /** * Input Validation */ @@ -77,16 +81,16 @@ export class ChannelsController { throw new Error('Referrer channel cannot be the same as the channel being verified.') } - // get user from userId - const user = await this.dynamodbService.users.get(userId) + // get user by channel Id + const user = await this.dynamodbService.users.get(id) - // ensure request's authorization code matches the user's authorization code - if (user.authorizationCode !== authorizationCode) { - throw new Error('Invalid request author. Permission denied.') + // ensure request is sent by the authorized actor + if (user.authorizationCode !== authorizationCode || user.youtubeVideoUrl !== youtubeVideoUrl) { + throw new Error('Authorization error. Either authorization code or video url is invalid.') } // ensure that Joystream channel exists - const jsChannel = await this.qnApi.getChannelById(joystreamChannelId.toString()) + const jsChannel = await this.qnApi.getChannelById(joystreamChannelId.toString()) // TODO: check if QN is lagging then what will happen if (!jsChannel) { throw new Error(`Joystream Channel ${joystreamChannelId} does not exist.`) } @@ -100,17 +104,14 @@ export class ChannelsController { } // get channel from user - let channel = await this.youtubeApi.getChannel(user) + let channel = await this.youtubeApi.operationalApi.getChannel(id) const existingChannel = await this.dynamodbService.repo.channels.get(channel.id) - // reset authorization code to prevent repeated save channel requests by authorization code re-use - const updatedUser: YtUser = { ...user, email, authorizationCode: randomBytes(10).toString('hex') } - const joystreamChannelLanguageIso = jsChannel.language?.iso // If channel already exists in the DB (in `OptedOut` state), then we // associate most properties of existing channel record with the new - // channel, i.e. createdAt, email. userId etc. and only override the + // channel, i.e. createdAt, email etc. and only override the // configuration properties provided in the request const updatedChannel: YtChannel = { ...(existingChannel @@ -120,7 +121,7 @@ export class ChannelsController { userAccessToken: channel.userAccessToken, userRefreshToken: channel.userRefreshToken, } - : { ...channel, email }), + : { ...channel }), joystreamChannelId, shouldBeIngested, videoCategoryId, @@ -129,10 +130,10 @@ export class ChannelsController { } // save user and channel - await this.saveUserAndChannel(updatedUser, updatedChannel) + await this.saveChannelAndVideos(updatedChannel) // return user and channel - return new SaveChannelResponse(new UserDto(updatedUser), new ChannelDto(updatedChannel)) + return new SaveChannelResponse(new ChannelDto(updatedChannel)) } catch (error) { const message = error instanceof Error ? error.message : error throw new BadRequestException(message) @@ -202,6 +203,7 @@ export class ChannelsController { } } + // TODO: on optout/suspend/disable ingestion remove all the polled not yet created videos, stop ingestion of channel's videos @Put(':joystreamChannelId/optout') @ApiBody({ type: OptoutChannelDto }) @ApiResponse({ type: ChannelDto }) @@ -232,7 +234,7 @@ export class ChannelsController { @ApiOperation({ description: `Updates given channel's videos category. Note: only channel owner can update the status`, }) - async updateCategoryChannel( + async updateChannelCategory( @Param('joystreamChannelId', ParseIntPipe) id: number, @Body() action: UpdateChannelCategoryDto ) { @@ -425,10 +427,7 @@ export class ChannelsController { }) } - private async saveUserAndChannel(user: YtUser, channel: YtChannel) { - // save user - await this.dynamodbService.users.save(user) - + private async saveChannelAndVideos(channel: YtChannel) { // save channel await this.dynamodbService.channels.save(channel) @@ -458,7 +457,7 @@ export class ChannelsController { const channel = await this.dynamodbService.channels.getByJoystreamId(joystreamChannelId) // Ensure channel is not suspended or opted out - if (YtChannel.isSuspended(channel) || channel.yppStatus === 'OptedOut') { + if (YtChannel.isSuspended(channel)) { throw new Error(`Can't perform "${actionType}" action on a "${channel.yppStatus}" channel. Permission denied.`) } diff --git a/src/services/httpApi/controllers/membership.ts b/src/services/httpApi/controllers/membership.ts index ed9ca16f..5ff896b6 100644 --- a/src/services/httpApi/controllers/membership.ts +++ b/src/services/httpApi/controllers/membership.ts @@ -13,25 +13,27 @@ export class MembershipController { constructor(@Inject('config') private config: ReadonlyConfig, private dynamodbService: DynamodbService) {} @ApiOperation({ - description: `Create Joystream's on-chain Membership for a verfifed YPP user. It will forward request - to Joystream faucet with an Authorization header to circumvent captcha verfication by the faucet`, + description: `Create Joystream's on-chain Membership for a verified YPP user. It will forward request + to Joystream faucet with an Authorization header to circumvent captcha verification by the faucet`, }) @ApiBody({ type: CreateMembershipRequest }) @ApiResponse({ type: CreateMembershipResponse }) @Post() - async createMembership(@Body() membershipParams: CreateMembershipRequest): Promise { + async createMembership(@Body() createMembershipParams: CreateMembershipRequest): Promise { try { - const { userId, authorizationCode, account, handle, avatar, about, name } = membershipParams + const { id, authorizationCode, youtubeVideoUrl, account, handle, avatar, about, name } = createMembershipParams // get user from userId - const user = await this.dynamodbService.users.get(userId) + const user = await this.dynamodbService.users.get(id) // ensure request's authorization code matches the user's authorization code - if (user.authorizationCode !== authorizationCode) { - throw new Error('Invalid request author. Permission denied.') + if (user.authorizationCode !== authorizationCode || user.youtubeVideoUrl !== youtubeVideoUrl) { + throw new Error('Authorization error. Either authorization code or video url is invalid.') } + + // Ensure that user has not already created a Joystream membership if (user.joystreamMemberId) { - throw new Error(`Already created Joysteam member ${user.joystreamMemberId} for user ${user.id}`) + throw new Error(`Already created Joystream member ${user.joystreamMemberId} for user ${user.id}`) } // send create membership request to faucet diff --git a/src/services/httpApi/controllers/status.ts b/src/services/httpApi/controllers/status.ts index 7581b379..f9cd2b05 100644 --- a/src/services/httpApi/controllers/status.ts +++ b/src/services/httpApi/controllers/status.ts @@ -30,7 +30,7 @@ export class StatusController { } = this.config const { totalCount: syncBacklog } = await this.contentProcessingService.getJobsCount() - return { version, syncStatus: enable ? 'enabled' : 'disabled', syncBacklog } + return { version, syncStatus: enable ? 'enabled' : 'disabled', syncBacklog, apiMode: this.config.youtube.apiMode } } catch (error) { const message = error instanceof Error ? error.message : error throw new NotFoundException(message) diff --git a/src/services/httpApi/controllers/users.ts b/src/services/httpApi/controllers/users.ts index 8342ade1..bd3fc0b8 100644 --- a/src/services/httpApi/controllers/users.ts +++ b/src/services/httpApi/controllers/users.ts @@ -1,15 +1,20 @@ import { BadRequestException, Body, Controller, Inject, Post } from '@nestjs/common' import { ApiBody, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' import { DynamodbService } from '../../../repository' +import { ReadonlyConfig } from '../../../types' import { ExitCodes, YoutubeApiError } from '../../../types/errors' import { YtChannel } from '../../../types/youtube' -import { IYoutubeApi } from '../../youtube/api' +import { YoutubeApi } from '../../youtube' import { VerifyChannelRequest, VerifyChannelResponse } from '../dtos' @Controller('users') @ApiTags('channels') export class UsersController { - constructor(@Inject('youtube') private youtube: IYoutubeApi, private dynamodbService: DynamodbService) {} + constructor( + private youtubeApi: YoutubeApi, + private dynamodbService: DynamodbService, + @Inject('config') private config: ReadonlyConfig + ) {} @ApiOperation({ description: `fetches user's channel from the supplied google authorization code, and verifies if it satisfies YPP induction criteria`, @@ -18,13 +23,14 @@ export class UsersController { @ApiResponse({ type: VerifyChannelResponse }) @Post() async verifyUserAndChannel( - @Body() { authorizationCode, youtubeRedirectUri }: VerifyChannelRequest + @Body() { authorizationCode, youtubeRedirectUri, youtubeVideoUrl }: VerifyChannelRequest ): Promise { try { - // get user from authorization code - const user = await this.youtube.getUserFromCode(authorizationCode, youtubeRedirectUri) + const user = authorizationCode // && this.config.youtube.apiMode !== 'api-free' + ? await this.youtubeApi.dataApiV3.getUserFromCode(authorizationCode, youtubeRedirectUri) + : await this.youtubeApi.ytdlp.getUserFromVideoUrl(youtubeVideoUrl!) - const [registeredChannel] = await this.dynamodbService.channels.getAll(user.id) + const registeredChannel = await this.dynamodbService.repo.channels.get(user.id) // Ensure 1. selected YT channel is not already registered for YPP program // OR 2. even if registered previously it has opted out. @@ -44,7 +50,9 @@ export class UsersController { } } - const { channel, errors } = await this.youtube.getVerifiedChannel(user) + const { channel, errors } = authorizationCode + ? await this.youtubeApi.dataApiV3.getVerifiedChannel(user) + : await this.youtubeApi.operationalApi.getVerifiedChannel(user) const whitelistedChannel = await this.dynamodbService.repo.whitelistChannels.get(channel.customUrl) // check if the channel is whitelisted @@ -60,11 +68,11 @@ export class UsersController { // return verified user return { - email: user.email, - userId: user.id, + id: user.id, + email: user.email || '', channelTitle: channel.title, channelDescription: channel.description, - avatarUrl: channel.thumbnails.medium, + avatarUrl: channel.thumbnails.high, bannerUrl: channel.bannerImageUrl, channelHandle: channel.customUrl, channelLanguage: channel.language, diff --git a/src/services/httpApi/controllers/videos.ts b/src/services/httpApi/controllers/videos.ts index e1daf82d..334734f6 100644 --- a/src/services/httpApi/controllers/videos.ts +++ b/src/services/httpApi/controllers/videos.ts @@ -3,7 +3,7 @@ import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' import { DynamodbService } from '../../../repository' import { YtVideo } from '../../../types/youtube' -@Controller('users/:userId/videos') +@Controller('channels/:channelId/videos') @ApiTags('channels') export class VideosController { constructor(private dynamodbService: DynamodbService) {} @@ -11,10 +11,10 @@ export class VideosController { @Get() @ApiResponse({ type: YtVideo, isArray: true }) @ApiOperation({ description: `Get videos across all channels owned by the user` }) - async get(@Param('userId') userId: string): Promise { + async get(@Param('channelId') channelId: string): Promise { try { // Get channels of the user - const channel = await this.dynamodbService.channels.getByUserId(userId) + const channel = await this.dynamodbService.channels.getById(channelId) // Get videos across all channels const result = await this.dynamodbService.repo.videos.query({ channelId: channel.id }, (q) => q) diff --git a/src/services/httpApi/dtos.ts b/src/services/httpApi/dtos.ts index 8ede2b75..13a8b098 100644 --- a/src/services/httpApi/dtos.ts +++ b/src/services/httpApi/dtos.ts @@ -13,7 +13,7 @@ import { ValidateIf, ValidateNested, } from 'class-validator' -import { Config } from '../../types' +import { Config, ReadonlyConfig } from '../../types' import { ExitCodes } from '../../types/errors' import { ChannelSyncStatus, @@ -24,11 +24,11 @@ import { TopReferrer, VideoState, YtChannel, - YtUser, YtVideo, channelYppStatus, } from '../../types/youtube' import { pluralizeNoun } from '../../utils/misc' +import { IsMutuallyExclusiveWith } from './utils' // NestJS Data Transfer Objects (DTO)s @@ -36,12 +36,12 @@ export class ThumbnailsDto { @ApiProperty() default: string @ApiProperty() medium: string @ApiProperty() high: string - @ApiProperty() standard: string } export class StatusDto { @ApiProperty() version: string @ApiProperty() syncStatus: 'enabled' | 'disabled' + @ApiProperty() apiMode: ReadonlyConfig['youtube']['apiMode'] @ApiProperty() syncBacklog: number } @@ -169,31 +169,32 @@ export class TopReferrerDto { } } -export class UserDto { - @ApiProperty() id: string - @ApiProperty() email: string - - constructor(user: YtUser) { - this.id = user.id - this.email = user.email - } -} - // Dto for verifying Youtube channel given the authorization code export class VerifyChannelRequest { - // Authorization code send to the backend after user o-auth verification - @IsString() @ApiProperty({ required: true }) authorizationCode: string - - @IsUrl({ require_tld: false }) @ApiProperty({ required: true }) youtubeRedirectUri: string + // Authorization code return from user o-auth response + @ValidateIf((v: VerifyChannelRequest) => v.youtubeRedirectUri !== undefined && v.youtubeVideoUrl === undefined) + @IsString() + @ApiProperty() + authorizationCode: string + + @ValidateIf((v: VerifyChannelRequest) => v.authorizationCode !== undefined && v.youtubeVideoUrl === undefined) + @IsUrl({ require_tld: false }) + @ApiProperty() + youtubeRedirectUri: string + + @ValidateIf((v: VerifyChannelRequest) => v.authorizationCode === undefined && v.youtubeRedirectUri === undefined) + @IsUrl({ require_tld: false }) + @ApiProperty() + youtubeVideoUrl: string } // Dto for verified Youtube channel response export class VerifyChannelResponse { // Email of the verified user - @IsEmail() @ApiProperty({ required: true }) email: string + @IsEmail() @ApiProperty() email: string - // ID of the verified user - @IsString() @ApiProperty({ required: true }) userId: string + // ID of the verified Youtube channel + @IsString() @ApiProperty({ required: true }) id: string // Youtube Channel/User handle @IsString() @ApiProperty() channelHandle: string @@ -216,14 +217,23 @@ export class VerifyChannelResponse { // Dto for saving the verified Youtube channel export class SaveChannelRequest { - // Authorization code send to the backend after user o-auth verification - @IsString() @ApiProperty({ required: true }) authorizationCode: string + // Authorization code return from user o-auth response + @ValidateIf((c: SaveChannelRequest) => c.youtubeVideoUrl === undefined) + @IsString() + @IsMutuallyExclusiveWith('youtubeVideoUrl') + @ApiProperty() + authorizationCode: string + + @ValidateIf((c: SaveChannelRequest) => c.authorizationCode === undefined) + @IsString() + @IsMutuallyExclusiveWith('authorizationCode') + @ApiProperty() + youtubeVideoUrl: string - // UserId of the Youtube creator return from Google Oauth API - @IsString() @ApiProperty({ required: true }) userId: string + // ID of the verified Youtube channel + @IsString() @ApiProperty({ required: true }) id: string - // Email of the user - @IsEmail() @ApiProperty({ required: true }) email: string + // TODO: do we need to ask for email? get from Orion // Joystream Channel ID of the user verifying his Youtube Channel for YPP @IsNumber() @ApiProperty({ required: true }) joystreamChannelId: number @@ -242,22 +252,28 @@ export class SaveChannelRequest { // Dto for save channel response export class SaveChannelResponse { - @ApiProperty() user: UserDto @ApiProperty({ type: ChannelDto }) channel: ChannelDto - constructor(user: UserDto, channel: ChannelDto) { - this.user = user + constructor(channel: ChannelDto) { this.channel = channel } } // Dto for creating membership request export class CreateMembershipRequest { - // UserId of the Youtube creator return from Google Oauth API - @IsString() @ApiProperty({ required: true }) userId: string + // ID of the verified Youtube channel + @IsString() @ApiProperty({ required: true }) id: string - // Authorization code send to the backend after user o-auth verification - @IsString() @ApiProperty({ required: true }) authorizationCode: string + // Authorization code return from user o-auth response + @ValidateIf((v: SaveChannelRequest) => v.authorizationCode === undefined) + @IsString() + @ApiProperty() + authorizationCode: string + + @ValidateIf((v: SaveChannelRequest) => v.authorizationCode === undefined) + @IsString() + @ApiProperty() + youtubeVideoUrl: string // Membership Account address @IsString() @ApiProperty({ required: true }) account: string diff --git a/src/services/httpApi/main.ts b/src/services/httpApi/main.ts index 4998220f..4c962788 100644 --- a/src/services/httpApi/main.ts +++ b/src/services/httpApi/main.ts @@ -15,7 +15,7 @@ import { QueryNodeApi } from '../query-node/api' import { RuntimeApi } from '../runtime/api' import { ContentProcessingService } from '../syncProcessing' import { YoutubePollingService } from '../syncProcessing/YoutubePollingService' -import { IYoutubeApi } from '../youtube/api' +import { YoutubeApi } from '../youtube' import { ChannelsController, StatusController, @@ -62,7 +62,7 @@ export async function bootstrapHttpApi( logging: LoggingService, runtimeApi: RuntimeApi, queryNodeApi: QueryNodeApi, - youtubeApi: IYoutubeApi, + youtubeApi: YoutubeApi, youtubePollingService: YoutubePollingService, contentProcessingService: ContentProcessingService ) { @@ -106,7 +106,7 @@ export async function bootstrapHttpApi( useValue: contentProcessingService, }, { - provide: 'youtube', + provide: YoutubeApi, useValue: youtubeApi, }, { diff --git a/src/services/httpApi/utils.ts b/src/services/httpApi/utils.ts new file mode 100644 index 00000000..823145b6 --- /dev/null +++ b/src/services/httpApi/utils.ts @@ -0,0 +1,22 @@ +import { ValidationArguments, ValidationOptions, registerDecorator } from 'class-validator' + +export function IsMutuallyExclusiveWith(property: string, validationOptions?: ValidationOptions) { + return function (object: any, propertyName: string) { + registerDecorator({ + name: 'isMutuallyExclusiveWith', + target: object.constructor, + propertyName: propertyName, + constraints: [property], + options: validationOptions, + validator: { + validate(value: any, args: ValidationArguments) { + const relatedValue = (args.object as any)[args.constraints[0]] + return value === null || value === undefined || relatedValue === null || relatedValue === undefined + }, + defaultMessage(args: ValidationArguments) { + return `${propertyName} is exclusive with ${args.constraints[0]}` + }, + }, + }) + } +} diff --git a/src/services/syncProcessing/ContentDownloadService.ts b/src/services/syncProcessing/ContentDownloadService.ts index bc6ccb9e..df8bbb29 100644 --- a/src/services/syncProcessing/ContentDownloadService.ts +++ b/src/services/syncProcessing/ContentDownloadService.ts @@ -11,7 +11,7 @@ import { ReadonlyConfig } from '../../types' import { DownloadJobData, DownloadJobOutput, VideoUnavailableReasons, YtChannel } from '../../types/youtube' import { restartEC2Instance } from '../../utils/restartEC2Instance' import { LoggingService } from '../logging' -import { IYoutubeApi } from '../youtube/api' +import { YoutubeApi } from '../youtube/' import { SyncUtils } from './utils' const exec = promisify(execCallback) @@ -24,7 +24,7 @@ export class ContentDownloadService { private config: Required & ReadonlyConfig['proxy'], logging: LoggingService, private dynamodbService: IDynamodbService, - private youtubeApi: IYoutubeApi + private youtubeApi: YoutubeApi ) { this.config = config this.logger = logging.createLogger('ContentDownloadService') @@ -87,7 +87,7 @@ export class ContentDownloadService { // download the video from youtube const { ext: fileExt } = await pTimeout( - this.youtubeApi.downloadVideo(video.url, this.config.downloadsDir), + this.youtubeApi.ytdlp.downloadVideo(video.url, this.config.downloadsDir), this.config.limits.pendingDownloadTimeoutSec * 1000, `Download timed-out` ) diff --git a/src/services/syncProcessing/YoutubePollingService.ts b/src/services/syncProcessing/YoutubePollingService.ts index ccfa154e..35a6c6d3 100644 --- a/src/services/syncProcessing/YoutubePollingService.ts +++ b/src/services/syncProcessing/YoutubePollingService.ts @@ -5,17 +5,17 @@ import { IDynamodbService } from '../../repository' import { YtChannel, YtDlpFlatPlaylistOutput, verifiedVariants } from '../../types/youtube' import { LoggingService } from '../logging' import { JoystreamClient } from '../runtime/client' -import { IYoutubeApi } from '../youtube/api' +import { YoutubeApi } from '../youtube' export class YoutubePollingService { private logger: Logger - private youtubeApi: IYoutubeApi + private youtubeApi: YoutubeApi private joystreamClient: JoystreamClient private dynamodbService: IDynamodbService public constructor( logging: LoggingService, - youtubeApi: IYoutubeApi, + youtubeApi: YoutubeApi, dynamodbService: IDynamodbService, joystreamClient: JoystreamClient ) { @@ -107,7 +107,7 @@ export class YoutubePollingService { channelsToBeIngested.map(async (ch) => { try { // ensure that channel exists on Youtube - await this.youtubeApi.ytdlpClient.ensureChannelExists(ch.id) + await this.youtubeApi.ytdlp.getChannel(ch.id) // ensure that Ypp collaborator member is still set as channel's collaborator const isCollaboratorSet = await this.joystreamClient.doesChannelHaveCollaborator(ch.joystreamChannelId) @@ -164,7 +164,7 @@ export class YoutubePollingService { const historicalVideosCountLimit = YtChannel.videoCap(channel) // get iDs of all sync-able videos within the channel limits - const videosIds = await this.youtubeApi.ytdlpClient.getVideosIDs(channel, historicalVideosCountLimit) + const videosIds = await this.youtubeApi.ytdlp.getVideosIDs(channel, historicalVideosCountLimit) // get all video Ids that are not yet being tracked let untrackedVideosIds = await this.getUntrackedVideosIds(channel, videosIds) @@ -175,7 +175,7 @@ export class YoutubePollingService { } // get all videos that are not yet being tracked - const untrackedVideos = await this.youtubeApi.ytdlpClient.getVideos( + const untrackedVideos = await this.youtubeApi.ytdlp.getVideos( channel, untrackedVideosIds.map((v) => v.id) ) diff --git a/src/services/syncProcessing/index.ts b/src/services/syncProcessing/index.ts index 4d07629a..0647e5c5 100644 --- a/src/services/syncProcessing/index.ts +++ b/src/services/syncProcessing/index.ts @@ -10,7 +10,7 @@ import { LoggingService } from '../logging' import { QueryNodeApi } from '../query-node/api' import { RuntimeApi } from '../runtime/api' import { JoystreamClient } from '../runtime/client' -import { IYoutubeApi } from '../youtube/api' +import { YoutubeApi } from '../youtube/' import { ContentCreationService } from './ContentCreationService' import { ContentDownloadService } from './ContentDownloadService' import { ContentMetadataService } from './ContentMetadataService' @@ -32,7 +32,7 @@ export class ContentProcessingService { private config: Required & ReadonlyConfig['endpoints'] & ReadonlyConfig['proxy'], logging: LoggingService, private dynamodbService: DynamodbService, - youtubeApi: IYoutubeApi, + youtubeApi: YoutubeApi, runtimeApi: RuntimeApi, private joystreamClient: JoystreamClient, queryNodeApi: QueryNodeApi From e446763ade4e10de33adc05b64693007983e036c Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Tue, 23 Jan 2024 23:25:02 +0500 Subject: [PATCH 07/17] remove Youtube Data V3 api integration & only support api-free signup --- config.yml | 6 - ...roperties-youtube-related-configuration.md | 56 -- src/app/index.ts | 2 +- src/repository/channel.ts | 8 - src/repository/user.ts | 23 +- src/schemas/config.ts | 35 +- src/services/httpApi/api-spec.json | 142 +---- src/services/httpApi/controllers/channels.ts | 27 +- src/services/httpApi/controllers/index.ts | 3 +- .../httpApi/controllers/membership.ts | 18 +- src/services/httpApi/controllers/status.ts | 33 +- src/services/httpApi/controllers/users.ts | 25 +- src/services/httpApi/controllers/youtube.ts | 37 -- src/services/httpApi/dtos.ts | 64 +-- src/services/httpApi/main.ts | 9 +- src/services/youtube/api.ts | 533 ------------------ src/services/youtube/index.ts | 12 +- src/services/youtube/openApi.ts | 14 +- src/services/youtube/operationalApi.ts | 11 +- src/types/errors.ts | 4 +- src/types/generated/ConfigJson.d.ts | 23 - src/types/index.ts | 12 +- src/types/youtube.ts | 30 +- 23 files changed, 89 insertions(+), 1038 deletions(-) delete mode 100644 src/services/httpApi/controllers/youtube.ts delete mode 100644 src/services/youtube/api.ts diff --git a/config.yml b/config.yml index 67fc6c89..fb82dd6b 100644 --- a/config.yml +++ b/config.yml @@ -34,12 +34,6 @@ logs: # username: elastic-username # password: elastic-password youtube: - apiMode: api - api: - clientId: google-client-id - clientSecret: google-client-secret - # adcKeyFilePath: path/to/adc-key-file.json - # maxAllowedQuotaUsageInPercentage: 95 operationalApi: url: http://127.0.0.1:8889 proxy: diff --git a/docs/config/definition-properties-youtube-related-configuration.md b/docs/config/definition-properties-youtube-related-configuration.md index 5ab91f8e..6f960c10 100644 --- a/docs/config/definition-properties-youtube-related-configuration.md +++ b/docs/config/definition-properties-youtube-related-configuration.md @@ -6,64 +6,8 @@ | Property | Type | Required | Nullable | Defined by | | :-------------------------------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [apiMode](#apimode) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/apiMode") | -| [api](#api) | `object` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api") | | [operationalApi](#operationalapi) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-operational-api-httpsgithubcombenjamin-loisonyoutube-operational-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/operationalApi") | -## apiMode - - - -`apiMode` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/apiMode") - -### apiMode Type - -`string` - -### apiMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :----------- | :---------- | -| `"api-free"` | | -| `"api"` | | -| `"both"` | | - -### apiMode Default Value - -The default value is: - -```json -"both" -``` - -## api - -Youtube API configuration - -`api` - -* is optional - -* Type: `object` ([Youtube API configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md)) - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api") - -### api Type - -`object` ([Youtube API configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md)) - ## operationalApi Youtube Operational API () configuration diff --git a/src/app/index.ts b/src/app/index.ts index 9623d108..e06f984e 100644 --- a/src/app/index.ts +++ b/src/app/index.ts @@ -32,7 +32,7 @@ export class Service { this.logger = this.logging.createLogger('Server') this.queryNodeApi = new QueryNodeApi(config.endpoints.queryNode, this.logging) this.dynamodbService = new DynamodbService(this.config.aws) - this.youtubeApi = new YoutubeApi(this.config, this.dynamodbService.repo.stats) + this.youtubeApi = new YoutubeApi(this.config) this.runtimeApi = new RuntimeApi(config.endpoints.joystreamNodeWs, this.logging) this.joystreamClient = new JoystreamClient(config, this.runtimeApi, this.queryNodeApi, this.logging) diff --git a/src/repository/channel.ts b/src/repository/channel.ts index c55a75f7..dd9135df 100644 --- a/src/repository/channel.ts +++ b/src/repository/channel.ts @@ -125,14 +125,6 @@ function createChannelModel(tablePrefix: ResourcePrefix) { // Banner or Background image URL bannerImageUrl: String, - // user access token obtained from authorization code after successful authentication - userAccessToken: String, - - // user refresh token that will be used to get new access token after expiration - userRefreshToken: String, - - uploadsPlaylistId: String, - // Should this channel be ingested for automated Youtube/Joystream syncing? shouldBeIngested: { type: Boolean, diff --git a/src/repository/user.ts b/src/repository/user.ts index df27ddfc..3b90cce8 100644 --- a/src/repository/user.ts +++ b/src/repository/user.ts @@ -15,30 +15,9 @@ function createUserModel(tablePrefix: ResourcePrefix) { hashKey: true, }, - // User email - email: String, - - // User youtube username - youtubeUsername: String, - - // User Google ID - googleId: String, - - // user authorization code - authorizationCode: String, - - // The URL for a specific video of Youtube channel with which the user is trying to register for YPP program + // The URL for a specific video of the Youtube channel with which the user verified for YPP youtubeVideoUrl: String, - // user access token obtained from authorization code after successful authentication - accessToken: String, - - // user refresh token that will be used to get new access token after expiration - refreshToken: String, - - // User avatar url - avatarUrl: String, - // joystream member ID for given creator. joystreamMemberId: Number, }, diff --git a/src/schemas/config.ts b/src/schemas/config.ts index 4a182e40..929e27e3 100644 --- a/src/schemas/config.ts +++ b/src/schemas/config.ts @@ -175,33 +175,6 @@ export const configSchema: JSONSchema7 = objectSchema({ title: 'Youtube related configuration', description: 'Youtube related configuration', properties: { - apiMode: { type: 'string', enum: ['api-free', 'api', 'both'], default: 'both' }, - api: objectSchema({ - title: 'Youtube API configuration', - description: 'Youtube API configuration', - properties: { - clientId: { type: 'string', description: 'Youtube Oauth2 Client Id' }, - clientSecret: { type: 'string', description: 'Youtube Oauth2 Client Secret' }, - maxAllowedQuotaUsageInPercentage: { - description: - `Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. ` + - `Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). ` + - `All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups.`, - type: 'number', - default: 95, - }, - adcKeyFilePath: { - type: 'string', - description: - `Path to the Google Cloud's Application Default Credentials (ADC) key file. ` + - `It is required to periodically monitor the Youtube API quota usage.`, - }, - }, - required: ['clientId', 'clientSecret'], - dependencies: { - maxAllowedQuotaUsageInPercentage: ['adcKeyFilePath'], - }, - }), operationalApi: objectSchema({ title: 'Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration', description: @@ -216,13 +189,7 @@ export const configSchema: JSONSchema7 = objectSchema({ }), }, - if: { - properties: { apiMode: { enum: ['api', 'both'] } }, - }, - then: { - required: ['api'], - }, - required: ['apiMode', 'operationalApi'], + required: ['operationalApi'], }), aws: objectSchema({ title: 'AWS configurations needed to connect with DynamoDB instance', diff --git a/src/services/httpApi/api-spec.json b/src/services/httpApi/api-spec.json index a1708d00..9b3c0ee0 100644 --- a/src/services/httpApi/api-spec.json +++ b/src/services/httpApi/api-spec.json @@ -240,7 +240,7 @@ }, "/channels/{joystreamChannelId}/category": { "put": { - "operationId": "ChannelsController_updateCategoryChannel", + "operationId": "ChannelsController_updateChannelCategory", "summary": "", "description": "Updates given channel's videos category. Note: only channel owner can update the status", "parameters": [ @@ -573,7 +573,7 @@ "post": { "operationId": "UsersController_verifyUserAndChannel", "summary": "", - "description": "fetches user's channel from the supplied google authorization code, and verifies if it satisfies YPP induction criteria", + "description": "fetches YT creator's channel from the provided Youtube video URL, and verifies if it satisfies YPP induction criteria", "parameters": [], "requestBody": { "required": true, @@ -602,57 +602,6 @@ ] } }, - "/youtube/quota-usage": { - "get": { - "operationId": "YoutubeController_getAll", - "summary": "", - "description": "Get youtube quota usage information", - "deprecated": true, - "parameters": [], - "responses": { - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Stats" - } - } - } - } - } - }, - "tags": [ - "youtube" - ] - } - }, - "/youtube/quota-usage/today": { - "get": { - "operationId": "YoutubeController_get", - "summary": "", - "description": "Get youtube quota usage information for today", - "deprecated": true, - "parameters": [], - "responses": { - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Stats" - } - } - } - } - }, - "tags": [ - "youtube" - ] - } - }, "/status": { "get": { "operationId": "StatusController_getStatus", @@ -676,55 +625,6 @@ ] } }, - "/status/quota-usage": { - "get": { - "operationId": "StatusController_getQuotaStats", - "summary": "", - "description": "Get youtube quota usage information", - "parameters": [], - "responses": { - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "type": "array", - "items": { - "$ref": "#/components/schemas/Stats" - } - } - } - } - } - }, - "tags": [ - "status" - ] - } - }, - "/status/quota-usage/today": { - "get": { - "operationId": "StatusController_getQuotaStatsForToday", - "summary": "", - "description": "Get youtube quota usage information for today", - "parameters": [], - "responses": { - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Stats" - } - } - } - } - }, - "tags": [ - "status" - ] - } - }, "/status/collaborator": { "get": { "operationId": "StatusController_getCollaboratorStatus", @@ -799,15 +699,15 @@ "SaveChannelRequest": { "type": "object", "properties": { - "authorizationCode": { - "type": "string" - }, "youtubeVideoUrl": { "type": "string" }, "id": { "type": "string" }, + "email": { + "type": "string" + }, "joystreamChannelId": { "type": "number" }, @@ -822,9 +722,9 @@ } }, "required": [ - "authorizationCode", "youtubeVideoUrl", "id", + "email", "joystreamChannelId", "shouldBeIngested", "videoCategoryId" @@ -1107,7 +1007,7 @@ "CHANNEL_NOT_FOUND", "VIDEO_NOT_FOUND", "VIDEO_PRIVACY_STATUS_NOT_UNLISTED", - "VIDEO_TITLE_NOT_MATCHING", + "VIDEO_TITLE_MISMATCH", "CHANNEL_ALREADY_REGISTERED", "CHANNEL_STATUS_SUSPENDED", "CHANNEL_CRITERIA_UNMET_SUBSCRIBERS", @@ -1166,28 +1066,17 @@ "VerifyChannelRequest": { "type": "object", "properties": { - "authorizationCode": { - "type": "string" - }, - "youtubeRedirectUri": { - "type": "string" - }, "youtubeVideoUrl": { "type": "string" } }, "required": [ - "authorizationCode", - "youtubeRedirectUri", "youtubeVideoUrl" ] }, "VerifyChannelResponse": { "type": "object", "properties": { - "email": { - "type": "string" - }, "id": { "type": "string" }, @@ -1211,7 +1100,6 @@ } }, "required": [ - "email", "id", "channelHandle", "channelTitle", @@ -1221,10 +1109,6 @@ "bannerUrl" ] }, - "Stats": { - "type": "object", - "properties": {} - }, "StatusDto": { "type": "object", "properties": { @@ -1234,9 +1118,6 @@ "syncStatus": { "type": "string" }, - "apiMode": { - "type": "object" - }, "syncBacklog": { "type": "number" } @@ -1244,19 +1125,19 @@ "required": [ "version", "syncStatus", - "apiMode", "syncBacklog" ] }, + "Stats": { + "type": "object", + "properties": {} + }, "CreateMembershipRequest": { "type": "object", "properties": { "id": { "type": "string" }, - "authorizationCode": { - "type": "string" - }, "youtubeVideoUrl": { "type": "string" }, @@ -1278,7 +1159,6 @@ }, "required": [ "id", - "authorizationCode", "youtubeVideoUrl", "account", "handle", diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts index eaff4cf6..d77d4113 100644 --- a/src/services/httpApi/controllers/channels.ts +++ b/src/services/httpApi/controllers/channels.ts @@ -55,23 +55,16 @@ export class ChannelsController { @ApiBody({ type: SaveChannelRequest }) @ApiResponse({ type: SaveChannelResponse }) @ApiOperation({ description: `Saves channel record of a YPP verified user` }) - // TODO: if channel exists dont allow to save again, async saveChannel(@Body() channelInfo: SaveChannelRequest): Promise { try { - const { - id, - authorizationCode, - youtubeVideoUrl, - joystreamChannelId, - shouldBeIngested, - videoCategoryId, - referrerChannelId, - } = channelInfo + const { id, youtubeVideoUrl, joystreamChannelId, shouldBeIngested, videoCategoryId, referrerChannelId } = + channelInfo - // Ensure that channel is not already registered for YPP program - if (await this.dynamodbService.repo.channels.get(id)) { - throw new Error('Channel already exists, cannot re-save the channel') - } + // TODO: check if needed. maybe add new flag `isSaved` to user record and check that instead + // // Ensure that channel is not already registered for YPP program + // if (await this.dynamodbService.repo.channels.get(id)) { + // throw new Error('Channel already exists, cannot re-save the channel') + // } /** * Input Validation @@ -85,8 +78,8 @@ export class ChannelsController { const user = await this.dynamodbService.users.get(id) // ensure request is sent by the authorized actor - if (user.authorizationCode !== authorizationCode || user.youtubeVideoUrl !== youtubeVideoUrl) { - throw new Error('Authorization error. Either authorization code or video url is invalid.') + if (user.youtubeVideoUrl !== youtubeVideoUrl) { + throw new Error('Authorization error. Youtube video url is invalid.') } // ensure that Joystream channel exists @@ -118,8 +111,6 @@ export class ChannelsController { ? { ...existingChannel, yppStatus: 'Unverified', - userAccessToken: channel.userAccessToken, - userRefreshToken: channel.userRefreshToken, } : { ...channel }), joystreamChannelId, diff --git a/src/services/httpApi/controllers/index.ts b/src/services/httpApi/controllers/index.ts index b95527fa..9a0ed8d5 100644 --- a/src/services/httpApi/controllers/index.ts +++ b/src/services/httpApi/controllers/index.ts @@ -1,5 +1,6 @@ export * from './channels' +export * from './membership' +export * from './referrers' export * from './status' export * from './users' export * from './videos' -export * from './youtube' diff --git a/src/services/httpApi/controllers/membership.ts b/src/services/httpApi/controllers/membership.ts index 5ff896b6..77a35842 100644 --- a/src/services/httpApi/controllers/membership.ts +++ b/src/services/httpApi/controllers/membership.ts @@ -21,14 +21,26 @@ export class MembershipController { @Post() async createMembership(@Body() createMembershipParams: CreateMembershipRequest): Promise { try { - const { id, authorizationCode, youtubeVideoUrl, account, handle, avatar, about, name } = createMembershipParams + const { id, youtubeVideoUrl, account, handle, avatar, about, name } = createMembershipParams + + // TODO: check if needed. maybe add new flag `isSaved` to user record and check that instead + // const registeredChannel = await this.dynamodbService.repo.channels.get(user.id) + // if (registeredChannel) { + // if (YtChannel.isVerified(registeredChannel) || registeredChannel.yppStatus === 'Unverified') { + // throw new YoutubeApiError( + // ExitCodes.YoutubeApi.CHANNEL_ALREADY_REGISTERED, + // `Cannot create membership for a channel already registered in YPP`, + // registeredChannel.joystreamChannelId + // ) + // } + // } // get user from userId const user = await this.dynamodbService.users.get(id) // ensure request's authorization code matches the user's authorization code - if (user.authorizationCode !== authorizationCode || user.youtubeVideoUrl !== youtubeVideoUrl) { - throw new Error('Authorization error. Either authorization code or video url is invalid.') + if (user.youtubeVideoUrl !== youtubeVideoUrl) { + throw new Error('Authorization error. Youtube video url is invalid.') } // Ensure that user has not already created a Joystream membership diff --git a/src/services/httpApi/controllers/status.ts b/src/services/httpApi/controllers/status.ts index f9cd2b05..27b2dfe6 100644 --- a/src/services/httpApi/controllers/status.ts +++ b/src/services/httpApi/controllers/status.ts @@ -1,7 +1,6 @@ import { Controller, Get, Inject, NotFoundException } from '@nestjs/common' import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' import BN from 'bn.js' -import { DynamodbService } from '../../../repository' import { ReadonlyConfig } from '../../../types' import { Stats } from '../../../types/youtube' import { RuntimeApi } from '../../runtime/api' @@ -12,7 +11,6 @@ import { CollaboratorStatusDto, StatusDto } from '../dtos' @ApiTags('status') export class StatusController { constructor( - private dynamodbService: DynamodbService, private runtimeApi: RuntimeApi, private contentProcessingService: ContentProcessingService, @Inject('config') private config: ReadonlyConfig @@ -23,41 +21,14 @@ export class StatusController { @ApiOperation({ description: `Get status info of YT-Synch service` }) async getStatus(): Promise { try { - // Get complete quota usage statss + // Get complete quota usage stats const { version, sync: { enable }, } = this.config const { totalCount: syncBacklog } = await this.contentProcessingService.getJobsCount() - return { version, syncStatus: enable ? 'enabled' : 'disabled', syncBacklog, apiMode: this.config.youtube.apiMode } - } catch (error) { - const message = error instanceof Error ? error.message : error - throw new NotFoundException(message) - } - } - - @Get('quota-usage') - @ApiResponse({ type: Stats, isArray: true }) - @ApiOperation({ description: `Get youtube quota usage information` }) - async getQuotaStats(): Promise { - try { - // Get complete quota usage statss - const stats = await this.dynamodbService.repo.stats.scan('partition', (s) => s) - return stats - } catch (error) { - const message = error instanceof Error ? error.message : error - throw new NotFoundException(message) - } - } - - @Get('quota-usage/today') - @ApiResponse({ type: Stats }) - @ApiOperation({ description: `Get youtube quota usage information for today` }) - async getQuotaStatsForToday(): Promise { - try { - const stats = this.dynamodbService.repo.stats.getOrSetTodaysStats() - return stats + return { version, syncStatus: enable ? 'enabled' : 'disabled', syncBacklog } } catch (error) { const message = error instanceof Error ? error.message : error throw new NotFoundException(message) diff --git a/src/services/httpApi/controllers/users.ts b/src/services/httpApi/controllers/users.ts index bd3fc0b8..8985fdd9 100644 --- a/src/services/httpApi/controllers/users.ts +++ b/src/services/httpApi/controllers/users.ts @@ -1,7 +1,6 @@ -import { BadRequestException, Body, Controller, Inject, Post } from '@nestjs/common' +import { BadRequestException, Body, Controller, Post } from '@nestjs/common' import { ApiBody, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' import { DynamodbService } from '../../../repository' -import { ReadonlyConfig } from '../../../types' import { ExitCodes, YoutubeApiError } from '../../../types/errors' import { YtChannel } from '../../../types/youtube' import { YoutubeApi } from '../../youtube' @@ -10,26 +9,17 @@ import { VerifyChannelRequest, VerifyChannelResponse } from '../dtos' @Controller('users') @ApiTags('channels') export class UsersController { - constructor( - private youtubeApi: YoutubeApi, - private dynamodbService: DynamodbService, - @Inject('config') private config: ReadonlyConfig - ) {} + constructor(private youtubeApi: YoutubeApi, private dynamodbService: DynamodbService) {} @ApiOperation({ - description: `fetches user's channel from the supplied google authorization code, and verifies if it satisfies YPP induction criteria`, + description: `fetches YT creator's channel from the provided Youtube video URL, and verifies if it satisfies YPP induction criteria`, }) @ApiBody({ type: VerifyChannelRequest }) @ApiResponse({ type: VerifyChannelResponse }) @Post() - async verifyUserAndChannel( - @Body() { authorizationCode, youtubeRedirectUri, youtubeVideoUrl }: VerifyChannelRequest - ): Promise { + async verifyUserAndChannel(@Body() { youtubeVideoUrl }: VerifyChannelRequest): Promise { try { - const user = authorizationCode // && this.config.youtube.apiMode !== 'api-free' - ? await this.youtubeApi.dataApiV3.getUserFromCode(authorizationCode, youtubeRedirectUri) - : await this.youtubeApi.ytdlp.getUserFromVideoUrl(youtubeVideoUrl!) - + const { user, video } = await this.youtubeApi.ytdlp.getUserAndVideoFromVideoUrl(youtubeVideoUrl) const registeredChannel = await this.dynamodbService.repo.channels.get(user.id) // Ensure 1. selected YT channel is not already registered for YPP program @@ -50,9 +40,7 @@ export class UsersController { } } - const { channel, errors } = authorizationCode - ? await this.youtubeApi.dataApiV3.getVerifiedChannel(user) - : await this.youtubeApi.operationalApi.getVerifiedChannel(user) + const { channel, errors } = await this.youtubeApi.operationalApi.getVerifiedChannel(video) const whitelistedChannel = await this.dynamodbService.repo.whitelistChannels.get(channel.customUrl) // check if the channel is whitelisted @@ -69,7 +57,6 @@ export class UsersController { // return verified user return { id: user.id, - email: user.email || '', channelTitle: channel.title, channelDescription: channel.description, avatarUrl: channel.thumbnails.high, diff --git a/src/services/httpApi/controllers/youtube.ts b/src/services/httpApi/controllers/youtube.ts deleted file mode 100644 index 2d485e9d..00000000 --- a/src/services/httpApi/controllers/youtube.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { Controller, Get, NotFoundException } from '@nestjs/common' -import { ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' -import { DynamodbService } from '../../../repository' -import { Stats } from '../../../types/youtube' - -@Controller('youtube') -@ApiTags('youtube') -export class YoutubeController { - constructor(private dynamodbService: DynamodbService) {} - - @Get('quota-usage') - @ApiResponse({ type: Stats, isArray: true }) - @ApiOperation({ description: `Get youtube quota usage information`, deprecated: true }) - async getAll(): Promise { - try { - // Get complete quota usage stats - const stats = await this.dynamodbService.repo.stats.scan('partition', (s) => s) - return stats - } catch (error) { - const message = error instanceof Error ? error.message : error - throw new NotFoundException(message) - } - } - - @Get('quota-usage/today') - @ApiResponse({ type: Stats }) - @ApiOperation({ description: `Get youtube quota usage information for today`, deprecated: true }) - async get(): Promise { - try { - const stats = this.dynamodbService.repo.stats.getOrSetTodaysStats() - return stats - } catch (error) { - const message = error instanceof Error ? error.message : error - throw new NotFoundException(message) - } - } -} diff --git a/src/services/httpApi/dtos.ts b/src/services/httpApi/dtos.ts index 13a8b098..0eaf11ad 100644 --- a/src/services/httpApi/dtos.ts +++ b/src/services/httpApi/dtos.ts @@ -13,7 +13,7 @@ import { ValidateIf, ValidateNested, } from 'class-validator' -import { Config, ReadonlyConfig } from '../../types' +import { Config } from '../../types' import { ExitCodes } from '../../types/errors' import { ChannelSyncStatus, @@ -28,7 +28,7 @@ import { channelYppStatus, } from '../../types/youtube' import { pluralizeNoun } from '../../utils/misc' -import { IsMutuallyExclusiveWith } from './utils' +import { YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP } from '../youtube' // NestJS Data Transfer Objects (DTO)s @@ -41,7 +41,6 @@ export class ThumbnailsDto { export class StatusDto { @ApiProperty() version: string @ApiProperty() syncStatus: 'enabled' | 'disabled' - @ApiProperty() apiMode: ReadonlyConfig['youtube']['apiMode'] @ApiProperty() syncBacklog: number } @@ -75,6 +74,16 @@ export class ChannelInductionRequirementsDto { constructor(requirements: Config['creatorOnboardingRequirements']) { this.requirements = [ + { + errorCode: ExitCodes.YoutubeApi.VIDEO_PRIVACY_STATUS_NOT_UNLISTED, + template: 'YouTube video should be {}.', + variables: ['Unlisted'], + }, + { + errorCode: ExitCodes.YoutubeApi.VIDEO_TITLE_MISMATCH, + template: 'YouTube video title should be {}.', + variables: [YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP], + }, { errorCode: ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_SUBSCRIBERS, template: 'YouTube channel has at least {}.', @@ -169,30 +178,16 @@ export class TopReferrerDto { } } -// Dto for verifying Youtube channel given the authorization code +// Dto for verifying Youtube channel given the Youtube video URL export class VerifyChannelRequest { - // Authorization code return from user o-auth response - @ValidateIf((v: VerifyChannelRequest) => v.youtubeRedirectUri !== undefined && v.youtubeVideoUrl === undefined) - @IsString() - @ApiProperty() - authorizationCode: string - - @ValidateIf((v: VerifyChannelRequest) => v.authorizationCode !== undefined && v.youtubeVideoUrl === undefined) - @IsUrl({ require_tld: false }) - @ApiProperty() - youtubeRedirectUri: string - - @ValidateIf((v: VerifyChannelRequest) => v.authorizationCode === undefined && v.youtubeRedirectUri === undefined) + // Youtube video URL required for the verification @IsUrl({ require_tld: false }) - @ApiProperty() + @ApiProperty({ required: true }) youtubeVideoUrl: string } // Dto for verified Youtube channel response export class VerifyChannelResponse { - // Email of the verified user - @IsEmail() @ApiProperty() email: string - // ID of the verified Youtube channel @IsString() @ApiProperty({ required: true }) id: string @@ -217,23 +212,16 @@ export class VerifyChannelResponse { // Dto for saving the verified Youtube channel export class SaveChannelRequest { - // Authorization code return from user o-auth response - @ValidateIf((c: SaveChannelRequest) => c.youtubeVideoUrl === undefined) - @IsString() - @IsMutuallyExclusiveWith('youtubeVideoUrl') - @ApiProperty() - authorizationCode: string - - @ValidateIf((c: SaveChannelRequest) => c.authorizationCode === undefined) - @IsString() - @IsMutuallyExclusiveWith('authorizationCode') - @ApiProperty() + // Youtube video URL required for the verification + @IsUrl({ require_tld: false }) + @ApiProperty({ required: true }) youtubeVideoUrl: string // ID of the verified Youtube channel @IsString() @ApiProperty({ required: true }) id: string - // TODO: do we need to ask for email? get from Orion + // Email of the YT user/channel + @IsEmail() @ApiProperty({ required: true }) email: string // Joystream Channel ID of the user verifying his Youtube Channel for YPP @IsNumber() @ApiProperty({ required: true }) joystreamChannelId: number @@ -264,15 +252,9 @@ export class CreateMembershipRequest { // ID of the verified Youtube channel @IsString() @ApiProperty({ required: true }) id: string - // Authorization code return from user o-auth response - @ValidateIf((v: SaveChannelRequest) => v.authorizationCode === undefined) - @IsString() - @ApiProperty() - authorizationCode: string - - @ValidateIf((v: SaveChannelRequest) => v.authorizationCode === undefined) - @IsString() - @ApiProperty() + // Youtube video URL required for the verification + @IsUrl({ require_tld: false }) + @ApiProperty({ required: true }) youtubeVideoUrl: string // Membership Account address diff --git a/src/services/httpApi/main.ts b/src/services/httpApi/main.ts index 4c962788..f22652c3 100644 --- a/src/services/httpApi/main.ts +++ b/src/services/httpApi/main.ts @@ -16,13 +16,7 @@ import { RuntimeApi } from '../runtime/api' import { ContentProcessingService } from '../syncProcessing' import { YoutubePollingService } from '../syncProcessing/YoutubePollingService' import { YoutubeApi } from '../youtube' -import { - ChannelsController, - StatusController, - UsersController, - VideosController, - YoutubeController, -} from './controllers' +import { ChannelsController, StatusController, UsersController, VideosController } from './controllers' import { MembershipController } from './controllers/membership' import { ReferrersController } from './controllers/referrers' @@ -80,7 +74,6 @@ export async function bootstrapHttpApi( ChannelsController, ReferrersController, UsersController, - YoutubeController, StatusController, MembershipController, ], diff --git a/src/services/youtube/api.ts b/src/services/youtube/api.ts deleted file mode 100644 index b4ad4edf..00000000 --- a/src/services/youtube/api.ts +++ /dev/null @@ -1,533 +0,0 @@ -import { MetricServiceClient } from '@google-cloud/monitoring' -import { youtube_v3 } from '@googleapis/youtube' -import { OAuth2Client } from 'google-auth-library' -import { GetTokenResponse } from 'google-auth-library/build/src/auth/oauth2client' -import { GaxiosError } from 'googleapis-common' -import { parse, toSeconds } from 'iso8601-duration' -import _ from 'lodash' -import moment from 'moment-timezone' -import { FetchError } from 'node-fetch' -import path from 'path' -import { StatsRepository } from '../../repository' -import { ReadonlyConfig, WithRequired, formattedJSON } from '../../types' -import { ExitCodes, YoutubeApiError } from '../../types/errors' -import { YtChannel, YtUser, YtVideo } from '../../types/youtube' -import { YtDlpClient } from './openApi' - -import Schema$Video = youtube_v3.Schema$Video -import Schema$Channel = youtube_v3.Schema$Channel - -interface IDataApiV3 { - getUserFromCode(code: string, youtubeRedirectUri: string): Promise - getChannel(user: Pick): Promise - getVerifiedChannel( - user: Pick - ): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> - getVideos(channel: YtChannel, ids: string[]): Promise -} - -interface IQuotaMonitoringDataApiV3 extends IDataApiV3 { - getQuotaUsage(): Promise - getQuotaLimit(): Promise -} - -class DataApiV3 implements IDataApiV3 { - private config: ReadonlyConfig - readonly ytdlpClient: YtDlpClient - - constructor(config: ReadonlyConfig) { - this.config = config - this.ytdlpClient = new YtDlpClient(config) - } - - private getAuth(youtubeRedirectUri?: string) { - if (this.config.youtube.apiMode !== 'api-free') { - return new OAuth2Client({ - clientId: this.config.youtube.api.clientId, - clientSecret: this.config.youtube.api.clientSecret, - redirectUri: youtubeRedirectUri, - }) - } else { - throw new Error('getAuth: Youtube API is not enabled') - } - } - - private getYoutube(accessToken?: string, refreshToken?: string) { - const auth = this.getAuth() - auth.setCredentials({ - access_token: accessToken, - refresh_token: refreshToken, - }) - return new youtube_v3.Youtube({ auth }) - } - - private async getAccessToken( - code: string, - youtubeRedirectUri: string - ): Promise> { - try { - const token = await this.getAuth(youtubeRedirectUri).getToken(code) - if (!token.tokens?.access_token) { - throw new Error('Access token not found in token response.') - } - - if (!token.tokens?.refresh_token) { - throw new Error( - 'Refresh token not found in token response. User authorization should be requested with `access_type: offline`' - ) - } - - return { access_token: token.tokens.access_token, refresh_token: token.tokens.refresh_token } - } catch (error) { - const message = error instanceof GaxiosError ? error.response?.data : error - throw new Error(`Could not get User's access token using authorization code ${formattedJSON(message)}`) - } - } - - async getUserFromCode(code: string, youtubeRedirectUri: string) { - const tokenResponse = await this.getAccessToken(code, youtubeRedirectUri) - - const tokenInfo = await this.getAuth().getTokenInfo(tokenResponse.access_token) - - if (!tokenInfo.sub) { - throw new Error( - `User id not found in token info. Please add required 'userinfo.profile' scope in user authorization request.` - ) - } - - if (!tokenInfo.email) { - throw new Error( - `User email not found in token info. Please add required 'userinfo.email' scope in user authorization request.` - ) - } - - const channel = await this.getChannel({ - id: tokenInfo.sub, - accessToken: tokenResponse.access_token, - refreshToken: tokenResponse.refresh_token, - }) - - const user: YtUser = { - id: channel.id, - email: tokenInfo.email, - accessToken: tokenResponse.access_token, - refreshToken: tokenResponse.refresh_token, - authorizationCode: code, - joystreamMemberId: undefined, - youtubeVideoUrl: undefined, - createdAt: new Date(), - } - return user - } - - async getChannel(user: Pick) { - const yt = this.getYoutube(user.accessToken, user.refreshToken) - - const channelResponse = await yt.channels - .list({ - part: ['snippet', 'contentDetails', 'statistics', 'brandingSettings'], - mine: true, - }) - .catch((err) => { - if (err instanceof FetchError && err.code === 'ENOTFOUND') { - throw new YoutubeApiError(ExitCodes.YoutubeApi.YOUTUBE_API_NOT_CONNECTED, err.message) - } - throw err - }) - - const [channel] = this.mapChannels(user, channelResponse.data.items ?? []) - - // Ensure channel exists - if (!channel) { - throw new YoutubeApiError(ExitCodes.YoutubeApi.CHANNEL_NOT_FOUND, `No Youtube Channel exists for given user`) - } - - return channel - } - - async getVerifiedChannel( - user: Pick - ): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> { - const { - minimumSubscribersCount, - minimumVideosCount, - minimumChannelAgeHours, - minimumVideoAgeHours, - minimumVideosPerMonth, - monthsToConsider, - } = this.config.creatorOnboardingRequirements - - const channel = await this.getChannel(user) - const errors: YoutubeApiError[] = [] - if (channel.statistics.subscriberCount < minimumSubscribersCount) { - errors.push( - new YoutubeApiError( - ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_SUBSCRIBERS, - `Channel ${channel.id} with ${channel.statistics.subscriberCount} subscribers does not ` + - `meet Youtube Partner Program requirement of ${minimumSubscribersCount} subscribers`, - channel.statistics.subscriberCount, - minimumSubscribersCount - ) - ) - } - - // at least 'minimumVideosCount' videos should be 'minimumVideoAgeHours' old - const videoCreationTimeCutoff = new Date() - videoCreationTimeCutoff.setHours(videoCreationTimeCutoff.getHours() - minimumVideoAgeHours) - - // filter all videos that are older than 'minimumVideoAgeHours' - const oldVideos = (await this.ytdlpClient.getVideosIDs(channel, minimumVideosCount, 'last')).filter( - (v) => v.publishedAt < videoCreationTimeCutoff - ) - if (oldVideos.length < minimumVideosCount) { - errors.push( - new YoutubeApiError( - ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_VIDEOS, - `Channel ${channel.id} with ${oldVideos.length} videos does not meet Youtube ` + - `Partner Program requirement of at least ${minimumVideosCount} videos, each ${( - minimumVideoAgeHours / 720 - ).toPrecision(2)} month old`, - channel.statistics.videoCount, - minimumVideosCount - ) - ) - } - - // TODO: make configurable (currently hardcoded to latest 1 month) - // at least 'minimumVideosPerMonth' should be there for 'monthsToConsider' - const nMonthsAgo = new Date() - nMonthsAgo.setMonth(nMonthsAgo.getMonth() - 1) - - const newVideos = (await this.ytdlpClient.getVideosIDs(channel, minimumVideosPerMonth)).filter( - (v) => v.publishedAt > nMonthsAgo - ) - if (newVideos.length < minimumVideosPerMonth) { - errors.push( - new YoutubeApiError( - ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_NEW_VIDEOS_REQUIREMENT, - `Channel ${channel.id} videos does not meet Youtube Partner Program ` + - `requirement of at least ${minimumVideosPerMonth} video per ` + - `month, posted over the last ${monthsToConsider} months`, - channel.statistics.videoCount, - minimumVideosPerMonth - ) - ) - } - - // Channel should be at least 'minimumChannelAgeHours' old - const channelCreationTimeCutoff = new Date() - channelCreationTimeCutoff.setHours(channelCreationTimeCutoff.getHours() - minimumChannelAgeHours) - if (new Date(channel.publishedAt) > channelCreationTimeCutoff) { - errors.push( - new YoutubeApiError( - ExitCodes.YoutubeApi.CHANNEL_CRITERIA_UNMET_CREATION_DATE, - `Channel ${channel.id} with creation time of ${channel.publishedAt} does not ` + - `meet Youtube Partner Program requirement of channel being at least ${( - minimumChannelAgeHours / 720 - ).toPrecision(2)} months old`, - channel.publishedAt, - channelCreationTimeCutoff - ) - ) - } - - return { channel, errors } - } - - async getVideos(channel: YtChannel, ids: string[]) { - const yt = this.getYoutube(channel.userAccessToken, channel.userRefreshToken) - try { - return this.iterateVideos(yt, channel, ids) - } catch (error) { - throw new Error(`Failed to fetch videos for channel ${channel.title}. Error: ${error}`) - } - } - - private async iterateVideos(youtube: youtube_v3.Youtube, channel: YtChannel, ids: string[]) { - let videos: YtVideo[] = [] - - // Youtube API allows to fetch up to 50 videos per request - const idsChunks = _.chunk(ids, 50) - - for (const idsChunk of idsChunks) { - const videosPage = idsChunk.length - ? ( - await youtube.videos - .list({ - id: idsChunk, - part: [ - 'id', - 'status', - 'snippet', - 'statistics', - 'fileDetails', - 'contentDetails', - 'liveStreamingDetails', - ], - }) - .catch((err) => { - if (err instanceof FetchError && err.code === 'ENOTFOUND') { - throw new YoutubeApiError(ExitCodes.YoutubeApi.YOUTUBE_API_NOT_CONNECTED, err.message) - } - throw err - }) - ).data?.items ?? [] - : [] - - const page = this.mapVideos(videosPage, channel) - videos = [...videos, ...page] - } - return videos - } - - private mapChannels(user: Pick, channels: Schema$Channel[]) { - return channels.map( - (channel) => - { - id: channel.id, - description: channel.snippet?.description, - title: channel.snippet?.title, - customUrl: channel.snippet?.customUrl, - userAccessToken: user.accessToken, - userRefreshToken: user.refreshToken, - thumbnails: { - default: channel.snippet?.thumbnails?.default?.url, - medium: channel.snippet?.thumbnails?.medium?.url, - high: channel.snippet?.thumbnails?.high?.url, - }, - statistics: { - viewCount: parseInt(channel.statistics?.viewCount ?? '0'), - subscriberCount: parseInt(channel.statistics?.subscriberCount ?? '0'), - videoCount: parseInt(channel.statistics?.videoCount ?? '0'), - commentCount: parseInt(channel.statistics?.commentCount ?? '0'), - }, - historicalVideoSyncedSize: 0, - bannerImageUrl: channel.brandingSettings?.image?.bannerExternalUrl, - uploadsPlaylistId: channel.contentDetails?.relatedPlaylists?.uploads, - language: channel.snippet?.defaultLanguage, - publishedAt: channel.snippet?.publishedAt, - performUnauthorizedSync: false, - shouldBeIngested: true, - allowOperatorIngestion: true, - yppStatus: 'Unverified', - createdAt: new Date(), - lastActedAt: new Date(), - phantomKey: 'phantomData', - } - ) - } - - private mapVideos(videos: Schema$Video[], channel: YtChannel): YtVideo[] { - return ( - videos - .map( - (video) => - { - id: video.id, - description: video.snippet?.description, - title: video.snippet?.title, - channelId: video.snippet?.channelId, - thumbnails: { - high: video.snippet?.thumbnails?.high?.url, - medium: video.snippet?.thumbnails?.medium?.url, - default: video.snippet?.thumbnails?.default?.url, - }, - url: `https://youtube.com/watch?v=${video.id}`, - publishedAt: video.snippet?.publishedAt, - createdAt: new Date(), - category: channel.videoCategoryId, - languageIso: channel.joystreamChannelLanguageIso, - privacyStatus: video.status?.privacyStatus, - ytRating: video.contentDetails?.contentRating?.ytRating, - liveBroadcastContent: video.snippet?.liveBroadcastContent, - license: video.status?.license, - duration: toSeconds(parse(video.contentDetails?.duration ?? 'PT0S')), - container: video.fileDetails?.container, - uploadStatus: video.status?.uploadStatus, - viewCount: parseInt(video.statistics?.viewCount ?? '0'), - state: 'New', - } - ) - // filter out videos that are not public, processed, have live-stream or age-restriction, since those can't be synced yet - .filter( - (v) => - v.uploadStatus === 'processed' && - v.privacyStatus === 'public' && - v.liveBroadcastContent === 'none' && - v.ytRating === undefined - ) - ) - } -} - -export class QuotaMonitoringDataApiV3 implements IQuotaMonitoringDataApiV3 { - private quotaMonitoringClient: MetricServiceClient | undefined - private googleCloudProjectId: string - private DEFAULT_MAX_ALLOWED_QUOTA_USAGE = 95 // 95% - private dataApiV3: IDataApiV3 - - constructor(private config: ReadonlyConfig, private statsRepo: StatsRepository) { - this.dataApiV3 = new DataApiV3(config) - - if (this.config.youtube.apiMode !== 'api-free') { - // Use the client id to get the google cloud project id - this.googleCloudProjectId = this.config.youtube.api?.clientId.split('-')[0] - - // if we have a key file path, we use it to authenticate with the google cloud monitoring api - if (this.config.youtube.api.adcKeyFilePath) { - this.quotaMonitoringClient = new MetricServiceClient({ - keyFile: path.resolve(this.config.youtube.api?.adcKeyFilePath), - }) - } - } else { - throw new Error(`QuotaMonitoringDataApiV3: Youtube API is not enabled`) - } - } - - async getQuotaLimit(): Promise { - // Get the unix timestamp for the start of the day in the PST timezone - const dateInPstZone = moment().tz('America/Los_Angeles') - dateInPstZone.startOf('day') - const pstUnixTimestamp = dateInPstZone.unix() - - const now = new Date() - - // Create the request - const request = { - name: this.quotaMonitoringClient?.projectPath(this.googleCloudProjectId), - filter: - `metric.type = "serviceruntime.googleapis.com/quota/limit" ` + - `AND ` + - `metric.labels.limit_name = defaultPerDayPerProject ` + - `AND ` + - `resource.labels.service= "youtube.googleapis.com"`, - interval: { - startTime: { - seconds: pstUnixTimestamp, - }, - endTime: { - seconds: Math.floor(now.getTime() / 1000), - }, - }, - } - - // Get time series metrics data - const timeSeries = await this.quotaMonitoringClient?.listTimeSeries(request) - - // Get Youtube API quota limit - const quotaLimit = Number((timeSeries![0][0]?.points || [])[0]?.value?.int64Value) - return _.isFinite(quotaLimit) ? quotaLimit : Number.MAX_SAFE_INTEGER - } - - async getQuotaUsage(): Promise { - // Get the unix timestamp for the start of the day in the PST timezone - const dateInPstZone = moment().tz('America/Los_Angeles') - dateInPstZone.startOf('day') - const pstUnixTimestamp = dateInPstZone.unix() - - const now: Date = new Date() - - // Create the request - const request = { - name: this.quotaMonitoringClient?.projectPath(this.googleCloudProjectId), - filter: - `metric.type = "serviceruntime.googleapis.com/quota/rate/net_usage" ` + - `AND ` + - `resource.labels.service= "youtube.googleapis.com" `, - interval: { - startTime: { - seconds: pstUnixTimestamp, - }, - endTime: { - seconds: Math.floor(now.getTime() / 1000), - }, - }, - } - - // Get time series metrics data - const timeSeries = await this.quotaMonitoringClient?.listTimeSeries(request) - - // Aggregate Youtube API quota usage for the day using the time series data - let quotaUsage = 0 - timeSeries![0].forEach((data) => { - quotaUsage = _.sumBy(data?.points, (point) => { - return Number(point.value?.int64Value) - }) - }) - - return quotaUsage - } - - getUserFromCode(code: string, youtubeRedirectUri: string) { - return this.dataApiV3.getUserFromCode(code, youtubeRedirectUri) - } - - async getVerifiedChannel(user: Pick) { - // These is no api quota check for this operation, as we allow untracked access to channel verification/signup endpoint. - const verifiedChannel = await this.dataApiV3.getVerifiedChannel(user) - - // increase used quota count by 1 because only one page is returned - await this.increaseUsedQuota({ signupQuotaIncrement: 1 }) - - return verifiedChannel - } - - async getChannel(user: Pick) { - // ensure have some left api quota - if (!(await this.canCallYoutube())) { - throw new YoutubeApiError( - ExitCodes.YoutubeApi.YOUTUBE_QUOTA_LIMIT_EXCEEDED, - 'No more quota left. Please try again later.' - ) - } - - // get channels from api - const channels = await this.dataApiV3.getChannel(user) - - // increase used quota count by 1 because only one page is returned - await this.increaseUsedQuota({ syncQuotaIncrement: 1 }) - - return channels - } - - async getVideos(channel: YtChannel, ids: string[]) { - // ensure have some left api quota - if (!(await this.canCallYoutube())) { - throw new YoutubeApiError( - ExitCodes.YoutubeApi.YOUTUBE_QUOTA_LIMIT_EXCEEDED, - 'No more quota left. Please try again later.' - ) - } - - // get videos from api - const videos = await this.dataApiV3.getVideos(channel, ids) - - // increase used quota count, 1 api call is being used per page of 50 videos - await this.increaseUsedQuota({ syncQuotaIncrement: Math.ceil(videos.length / 50) }) - - return videos - } - - private async increaseUsedQuota({ syncQuotaIncrement = 0, signupQuotaIncrement = 0 }) { - // Quota resets at Pacific Time, and pst is 8 hours behind UTC - const stats = await this.statsRepo.getOrSetTodaysStats() - const statsModel = await this.statsRepo.getModel() - - await statsModel.update( - { partition: 'stats', date: stats.date }, - { $ADD: { syncQuotaUsed: syncQuotaIncrement, signupQuotaUsed: signupQuotaIncrement } } - ) - } - - private async canCallYoutube(): Promise { - if (this.config.youtube.apiMode !== 'api-free' && this.quotaMonitoringClient) { - const quotaUsage = await this.getQuotaUsage() - const quotaLimit = await this.getQuotaLimit() - return ( - (quotaUsage * 100) / quotaLimit < - (this.config.youtube.api?.maxAllowedQuotaUsageInPercentage || this.DEFAULT_MAX_ALLOWED_QUOTA_USAGE) - ) - } - return true - } -} diff --git a/src/services/youtube/index.ts b/src/services/youtube/index.ts index 2c079733..0ef11e8e 100644 --- a/src/services/youtube/index.ts +++ b/src/services/youtube/index.ts @@ -1,22 +1,16 @@ -import { StatsRepository } from '../../repository' import { ReadonlyConfig } from '../../types' -import { QuotaMonitoringDataApiV3 } from './api' import { YtDlpClient } from './openApi' import { YoutubeOperationalApi } from './operationalApi' +export const YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP = `I want to be in YPP` + export class YoutubeApi { - public readonly dataApiV3: QuotaMonitoringDataApiV3 public readonly ytdlp: YtDlpClient public readonly operationalApi: YoutubeOperationalApi - constructor(private config: ReadonlyConfig, statsRepo: StatsRepository) { + constructor(private config: ReadonlyConfig) { this.ytdlp = new YtDlpClient(config) - this.operationalApi = new YoutubeOperationalApi(config) - - if (this.config.youtube.apiMode !== 'api-free') { - this.dataApiV3 = new QuotaMonitoringDataApiV3(config, statsRepo) - } } getCreatorOnboardingRequirements() { diff --git a/src/services/youtube/openApi.ts b/src/services/youtube/openApi.ts index fe48fcce..42aea778 100644 --- a/src/services/youtube/openApi.ts +++ b/src/services/youtube/openApi.ts @@ -13,7 +13,7 @@ export interface IOpenYTApi { getVideoFromUrl(videoUrl: string): Promise getVideos(channel: YtChannel, ids: string[]): Promise downloadVideo(videoUrl: string, outPath: string): ReturnType - getUserFromVideoUrl(videoUrl: string): Promise + getUserAndVideoFromVideoUrl(videoUrl: string): Promise<{ user: YtUser; video: YtVideo }> } export class YtDlpClient implements IOpenYTApi { @@ -40,20 +40,16 @@ export class YtDlpClient implements IOpenYTApi { return this.mapVideo(output) } - async getUserFromVideoUrl(videoUrl: string): Promise { - const { stdout } = await this.exec(`${this.ytdlpPath} --print channel_id ${videoUrl}`) + async getUserAndVideoFromVideoUrl(videoUrl: string): Promise<{ user: YtUser; video: YtVideo }> { + const video = await this.getVideoFromUrl(videoUrl) const user: YtUser = { - id: stdout.replace(/(\r\n|\n|\r)/gm, ''), - email: undefined, - accessToken: undefined, - refreshToken: undefined, - authorizationCode: undefined, + id: video.channelId, joystreamMemberId: undefined, youtubeVideoUrl: videoUrl, createdAt: new Date(), } - return user + return { user, video } } async downloadVideo(videoUrl: string, outPath: string): ReturnType { diff --git a/src/services/youtube/operationalApi.ts b/src/services/youtube/operationalApi.ts index 371aafb5..e9e50af5 100644 --- a/src/services/youtube/operationalApi.ts +++ b/src/services/youtube/operationalApi.ts @@ -1,7 +1,8 @@ import axios from 'axios' +import { YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP } from '.' import { ReadonlyConfig } from '../../types' import { ExitCodes, YoutubeApiError } from '../../types/errors' -import { YtChannel, YtUser } from '../../types/youtube' +import { YtChannel, YtVideo } from '../../types/youtube' import { YtDlpClient } from './openApi' type OperationalApiChannelListResponse = { @@ -39,8 +40,6 @@ type Image = { url: string } -const YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP = `I want to be in YPP` - export class YoutubeOperationalApi { private baseUrl: string readonly ytdlpClient: YtDlpClient @@ -66,7 +65,7 @@ export class YoutubeOperationalApi { } } - async getVerifiedChannel(user: YtUser): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> { + async getVerifiedChannel(video: YtVideo): Promise<{ channel: YtChannel; errors: YoutubeApiError[] }> { const { minimumSubscribersCount, minimumVideosCount, @@ -78,8 +77,6 @@ export class YoutubeOperationalApi { const errors: YoutubeApiError[] = [] - const video = await this.ytdlpClient.getVideoFromUrl(user.youtubeVideoUrl!) - if (video.privacyStatus !== 'unlisted') { errors.push( new YoutubeApiError( @@ -94,7 +91,7 @@ export class YoutubeOperationalApi { if (video.title !== YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP) { errors.push( new YoutubeApiError( - ExitCodes.YoutubeApi.VIDEO_TITLE_NOT_MATCHING, + ExitCodes.YoutubeApi.VIDEO_TITLE_MISMATCH, `Video ${video.id} title is not matching`, video.title, YT_VIDEO_TITLE_REQUIRED_FOR_SIGNUP diff --git a/src/types/errors.ts b/src/types/errors.ts index 06882082..7f0cfc3c 100644 --- a/src/types/errors.ts +++ b/src/types/errors.ts @@ -2,8 +2,8 @@ export namespace ExitCodes { export enum YoutubeApi { CHANNEL_NOT_FOUND = 'CHANNEL_NOT_FOUND', VIDEO_NOT_FOUND = 'VIDEO_NOT_FOUND', - VIDEO_PRIVACY_STATUS_NOT_UNLISTED = 'VIDEO_PRIVACY_STATUS_NOT_UNLISTED', // TODO: Add this to the error message - VIDEO_TITLE_NOT_MATCHING = 'VIDEO_TITLE_NOT_MATCHING', // TODO: Add this to the error message + VIDEO_PRIVACY_STATUS_NOT_UNLISTED = 'VIDEO_PRIVACY_STATUS_NOT_UNLISTED', + VIDEO_TITLE_MISMATCH = 'VIDEO_TITLE_MISMATCH', CHANNEL_ALREADY_REGISTERED = 'CHANNEL_ALREADY_REGISTERED', CHANNEL_STATUS_SUSPENDED = 'CHANNEL_STATUS_SUSPENDED', CHANNEL_CRITERIA_UNMET_SUBSCRIBERS = 'CHANNEL_CRITERIA_UNMET_SUBSCRIBERS', diff --git a/src/types/generated/ConfigJson.d.ts b/src/types/generated/ConfigJson.d.ts index 3ae061d8..61266bd5 100644 --- a/src/types/generated/ConfigJson.d.ts +++ b/src/types/generated/ConfigJson.d.ts @@ -187,31 +187,8 @@ export interface ElasticsearchAuthenticationOptions { * Youtube related configuration */ export interface YoutubeRelatedConfiguration { - apiMode: 'api-free' | 'api' | 'both' - api?: YoutubeAPIConfiguration operationalApi: YoutubeOperationalAPIHttpsGithubComBenjaminLoisonYouTubeOperationalAPIConfiguration } -/** - * Youtube API configuration - */ -export interface YoutubeAPIConfiguration { - /** - * Youtube Oauth2 Client Id - */ - clientId: string - /** - * Youtube Oauth2 Client Secret - */ - clientSecret: string - /** - * Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups. - */ - maxAllowedQuotaUsageInPercentage?: number - /** - * Path to the Google Cloud's Application Default Credentials (ADC) key file. It is required to periodically monitor the Youtube API quota usage. - */ - adcKeyFilePath?: string -} /** * Youtube Operational API (https://github.com/Benjamin-Loison/YouTube-operational-API) configuration */ diff --git a/src/types/index.ts b/src/types/index.ts index 2c2c01eb..f91863f0 100644 --- a/src/types/index.ts +++ b/src/types/index.ts @@ -14,19 +14,9 @@ type SyncDisabled = Omit, 'lim limits?: Omit, 'storage'> & { storage: number } } -type YoutubeApiEnabled = Omit & { - apiMode: 'api' | 'both' - api: NonNullable -} - -type YoutubeApiDisabled = Omit & { - apiMode: 'api-free' -} - -export type Config = Omit & { +export type Config = Omit & { version: string sync: SyncEnabled | SyncDisabled - youtube: YoutubeApiEnabled | YoutubeApiDisabled } export type ReadonlyConfig = DeepReadonly diff --git a/src/types/youtube.ts b/src/types/youtube.ts index bd07bf4d..011d7a5d 100644 --- a/src/types/youtube.ts +++ b/src/types/youtube.ts @@ -68,15 +68,6 @@ export class YtChannel { // total size of historical videos synced (videos that were published on Youtube before YPP signup) historicalVideoSyncedSize: number - // Channel owner's access token - userAccessToken: string - - // Channel owner's refresh token - userRefreshToken: string - - // Channel's playlist ID - uploadsPlaylistId: string - // Should this channel be ingested for automated Youtube/Joystream syncing? shouldBeIngested: boolean @@ -165,29 +156,12 @@ export class YtChannel { } } -/** - * We use the same DynamoDB table for storing users/channels verified through - * Oauth api vs non-api (i.e. through shared a URL for a required video). - */ export class YtUser { // Youtube channel ID id: string - // Youtube User/Channel email (will only be available for users signed up through api workflow) - email: string | undefined - - // User access token (will only be available for users signed up through api workflow) - accessToken: string | undefined - - // User refresh token (will only be available for users signed up through api workflow) - refreshToken: string | undefined - - // User authorization code (will only be available for users signed up through api workflow) - authorizationCode: string | undefined - - // The URL for a specific video of Youtube channel with which the user is trying to register - // for YPP program (will only be available for users signed up through api-free workflow) - youtubeVideoUrl: string | undefined + // The URL for a specific video of Youtube channel with which the user verified for YPP + youtubeVideoUrl: string // Corresponding Joystream member ID for Youtube user joystreamMemberId: number | undefined From 17fd2a055b806997670703ac1396e365e4b6f5ad Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Tue, 23 Jan 2024 23:45:29 +0500 Subject: [PATCH 08/17] update yarn.lock file --- yarn.lock | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/yarn.lock b/yarn.lock index 7023753e..96135267 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7783,6 +7783,16 @@ get-caller-file@^2.0.1, get-caller-file@^2.0.5: resolved "https://registry.yarnpkg.com/get-caller-file/-/get-caller-file-2.0.5.tgz#4f94412a82db32f36e3b0b9741f8a97feb031f7e" integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== +get-graphql-schema@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/get-graphql-schema/-/get-graphql-schema-2.1.2.tgz#ffa418534224a75cd7afc8f87b70109ca9ec3fe9" + integrity sha512-1z5Hw91VrE3GrpCZE6lE8Dy+jz4kXWesLS7rCSjwOxf5BOcIedAZeTUJRIeIzmmR+PA9CKOkPTYFRJbdgUtrxA== + dependencies: + chalk "^2.4.1" + graphql "^14.0.2" + minimist "^1.2.0" + node-fetch "^2.2.0" + get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" @@ -8154,7 +8164,7 @@ graphql-ws@^4.4.1: resolved "https://registry.yarnpkg.com/graphql-ws/-/graphql-ws-4.9.0.tgz#5cfd8bb490b35e86583d8322f5d5d099c26e365c" integrity sha512-sHkK9+lUm20/BGawNEWNtVAeJzhZeBg21VmvmLoT5NdGVeZWv5PdIhkcayQIAgjSyyQ17WMKmbDijIPG2On+Ag== -graphql@^14.7.0: +graphql@^14.0.2, graphql@^14.7.0: version "14.7.0" resolved "https://registry.yarnpkg.com/graphql/-/graphql-14.7.0.tgz#7fa79a80a69be4a31c27dda824dc04dac2035a72" integrity sha512-l0xWZpoPKpppFzMfvVyFmp9vLN7w/ZZJPefUicMCepfJeQ8sMcztloGYY9DfjVPo6tIUDzU5Hw3MUbIjj9AVVA== @@ -11072,6 +11082,13 @@ node-fetch@2.6.7, node-fetch@^2.6.1, node-fetch@^2.6.7: dependencies: whatwg-url "^5.0.0" +node-fetch@^2.2.0: + version "2.7.0" + resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.7.0.tgz#d0f0fa6e3e2dc1d27efcd8ad99d550bda94d187d" + integrity sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A== + dependencies: + whatwg-url "^5.0.0" + node-fetch@^2.6.0: version "2.6.9" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.9.tgz#7c7f744b5cc6eb5fd404e0c7a9fec630a55657e6" From a55308136c03b6233c78664e28bc5282557fcf10 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Wed, 24 Jan 2024 20:44:13 +0500 Subject: [PATCH 09/17] fix: dont reupload data object if it already exists on Storage Node --- src/services/storage-node/api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/storage-node/api.ts b/src/services/storage-node/api.ts index 1097ac5d..40dae0b8 100644 --- a/src/services/storage-node/api.ts +++ b/src/services/storage-node/api.ts @@ -74,7 +74,7 @@ export class StorageNodeApi { const storageNodeUrl = error.config?.url const { status, data } = error.response - if (data?.message?.includes(`Data object ${dataObjectId} has already been accepted by storage node`)) { + if (data?.message?.includes(`Data object ${dataObjectId} already exist`)) { // No need to throw an error, we can continue with the next asset continue } From 0867a20a9a0f8b71709dcbe462b2fdbbe3a58f0b Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 6 Jun 2024 11:14:35 +0500 Subject: [PATCH 10/17] remove POST /membership endpoint from httpApi app --- config.yml | 3 - docker-compose.yml | 2 - docs/cli/help.md | 2 +- .../config/definition-properties-joystream.md | 19 ----- src/repository/user.ts | 3 - src/schemas/config.ts | 13 +-- src/services/httpApi/api-spec.json | 83 ------------------- src/services/httpApi/controllers/index.ts | 1 - .../httpApi/controllers/membership.ts | 81 ------------------ src/services/httpApi/controllers/users.ts | 6 -- src/services/httpApi/dtos.ts | 40 --------- src/services/httpApi/main.ts | 10 +-- src/services/youtube/openApi.ts | 1 - src/types/errors.ts | 9 -- src/types/generated/ConfigJson.d.ts | 13 --- src/types/youtube.ts | 19 +---- 16 files changed, 5 insertions(+), 300 deletions(-) delete mode 100644 src/services/httpApi/controllers/membership.ts diff --git a/config.yml b/config.yml index fb82dd6b..95c20720 100644 --- a/config.yml +++ b/config.yml @@ -50,9 +50,6 @@ httpApi: port: 3001 ownerKey: ypp-owner-key joystream: - faucet: - endpoint: http://localhost:3002/register - captchaBypassKey: some-random-key app: name: app-name accountSeed: 'example_string_seed' diff --git a/docker-compose.yml b/docker-compose.yml index 882d5b08..4d8648d8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,8 +40,6 @@ services: # YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID: ${YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID} # YT_SYNCH__JOYSTREAM__APP__NAME: ${YT_SYNCH__JOYSTREAM__APP__NAME} # YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED: ${YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED} - # YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT: ${YT_SYNCH__JOYSTREAM__FAUCET__ENDPOINT} - # YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY: ${YT_SYNCH__JOYSTREAM__FAUCET__CAPTCHA_BYPASS_KEY} ports: - 127.0.0.1:${YT_SYNCH__HTTP_API__PORT}:${YT_SYNCH__HTTP_API__PORT} networks: diff --git a/docs/cli/help.md b/docs/cli/help.md index 1c92236a..9fff5d1d 100644 --- a/docs/cli/help.md +++ b/docs/cli/help.md @@ -20,4 +20,4 @@ OPTIONS --all see all commands in CLI ``` -_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v3.3.1/src/commands/help.ts)_ +_See code: [@oclif/plugin-help](https://github.com/oclif/plugin-help/blob/v3.2.18/src/commands/help.ts)_ diff --git a/docs/config/definition-properties-joystream.md b/docs/config/definition-properties-joystream.md index c7b016db..56aab7c2 100644 --- a/docs/config/definition-properties-joystream.md +++ b/docs/config/definition-properties-joystream.md @@ -6,28 +6,9 @@ | Property | Type | Required | Nullable | Defined by | | :------------------------------------------ | :------- | :------- | :------------- | :----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [faucet](#faucet) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet") | | [app](#app) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-app.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/app") | | [channelCollaborator](#channelcollaborator) | `object` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-joystream-channel-collaborator-used-for-syncing-the-content.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/channelCollaborator") | -## faucet - -Joystream's faucet configuration (needed for captcha-free membership creation) - -`faucet` - -* is required - -* Type: `object` ([Details](definition-properties-joystream-properties-faucet.md)) - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet") - -### faucet Type - -`object` ([Details](definition-properties-joystream-properties-faucet.md)) - ## app Joystream Metaprotocol App specific configuration diff --git a/src/repository/user.ts b/src/repository/user.ts index 3b90cce8..102ad274 100644 --- a/src/repository/user.ts +++ b/src/repository/user.ts @@ -17,9 +17,6 @@ function createUserModel(tablePrefix: ResourcePrefix) { // The URL for a specific video of the Youtube channel with which the user verified for YPP youtubeVideoUrl: String, - - // joystream member ID for given creator. - joystreamMemberId: Number, }, { saveUnknown: false, diff --git a/src/schemas/config.ts b/src/schemas/config.ts index 929e27e3..30eec768 100644 --- a/src/schemas/config.ts +++ b/src/schemas/config.ts @@ -20,17 +20,6 @@ export const configSchema: JSONSchema7 = objectSchema({ joystream: objectSchema({ description: 'Joystream network related configuration', properties: { - faucet: objectSchema({ - description: `Joystream's faucet configuration (needed for captcha-free membership creation)`, - properties: { - endpoint: { type: 'string', description: `Joystream's faucet URL` }, - captchaBypassKey: { - type: 'string', - description: `Bearer Authentication Key needed to bypass captcha verification on Faucet`, - }, - }, - required: ['endpoint', 'captchaBypassKey'], - }), app: objectSchema({ description: 'Joystream Metaprotocol App specific configuration', properties: { @@ -78,7 +67,7 @@ export const configSchema: JSONSchema7 = objectSchema({ required: ['memberId', 'account'], }), }, - required: ['faucet', 'app', 'channelCollaborator'], + required: ['app', 'channelCollaborator'], }), endpoints: objectSchema({ description: 'Specifies external endpoints that the distributor node will connect to', diff --git a/src/services/httpApi/api-spec.json b/src/services/httpApi/api-spec.json index 9b3c0ee0..1bd1119d 100644 --- a/src/services/httpApi/api-spec.json +++ b/src/services/httpApi/api-spec.json @@ -647,39 +647,6 @@ "status" ] } - }, - "/membership": { - "post": { - "operationId": "MembershipController_createMembership", - "summary": "", - "description": "Create Joystream's on-chain Membership for a verified YPP user. It will forward request\n to Joystream faucet with an Authorization header to circumvent captcha verification by the faucet", - "parameters": [], - "requestBody": { - "required": true, - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateMembershipRequest" - } - } - } - }, - "responses": { - "default": { - "description": "", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/CreateMembershipResponse" - } - } - } - } - }, - "tags": [ - "membership" - ] - } } }, "info": { @@ -1131,56 +1098,6 @@ "Stats": { "type": "object", "properties": {} - }, - "CreateMembershipRequest": { - "type": "object", - "properties": { - "id": { - "type": "string" - }, - "youtubeVideoUrl": { - "type": "string" - }, - "account": { - "type": "string" - }, - "handle": { - "type": "string" - }, - "avatar": { - "type": "string" - }, - "about": { - "type": "string" - }, - "name": { - "type": "string" - } - }, - "required": [ - "id", - "youtubeVideoUrl", - "account", - "handle", - "avatar", - "about", - "name" - ] - }, - "CreateMembershipResponse": { - "type": "object", - "properties": { - "memberId": { - "type": "number" - }, - "handle": { - "type": "string" - } - }, - "required": [ - "memberId", - "handle" - ] } } } diff --git a/src/services/httpApi/controllers/index.ts b/src/services/httpApi/controllers/index.ts index 9a0ed8d5..fcf7b68a 100644 --- a/src/services/httpApi/controllers/index.ts +++ b/src/services/httpApi/controllers/index.ts @@ -1,5 +1,4 @@ export * from './channels' -export * from './membership' export * from './referrers' export * from './status' export * from './users' diff --git a/src/services/httpApi/controllers/membership.ts b/src/services/httpApi/controllers/membership.ts deleted file mode 100644 index 77a35842..00000000 --- a/src/services/httpApi/controllers/membership.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { BadRequestException, Body, Controller, Inject, Post } from '@nestjs/common' -import { ApiBody, ApiOperation, ApiResponse, ApiTags } from '@nestjs/swagger' -import axios from 'axios' -import { DynamodbService } from '../../../repository' -import { ReadonlyConfig } from '../../../types' -import { FaucetApiError } from '../../../types/errors' -import { FaucetRegisterMembershipParams, FaucetRegisterMembershipResponse } from '../../../types/youtube' -import { CreateMembershipRequest, CreateMembershipResponse } from '../dtos' - -@Controller('membership') -@ApiTags('membership') -export class MembershipController { - constructor(@Inject('config') private config: ReadonlyConfig, private dynamodbService: DynamodbService) {} - - @ApiOperation({ - description: `Create Joystream's on-chain Membership for a verified YPP user. It will forward request - to Joystream faucet with an Authorization header to circumvent captcha verification by the faucet`, - }) - @ApiBody({ type: CreateMembershipRequest }) - @ApiResponse({ type: CreateMembershipResponse }) - @Post() - async createMembership(@Body() createMembershipParams: CreateMembershipRequest): Promise { - try { - const { id, youtubeVideoUrl, account, handle, avatar, about, name } = createMembershipParams - - // TODO: check if needed. maybe add new flag `isSaved` to user record and check that instead - // const registeredChannel = await this.dynamodbService.repo.channels.get(user.id) - // if (registeredChannel) { - // if (YtChannel.isVerified(registeredChannel) || registeredChannel.yppStatus === 'Unverified') { - // throw new YoutubeApiError( - // ExitCodes.YoutubeApi.CHANNEL_ALREADY_REGISTERED, - // `Cannot create membership for a channel already registered in YPP`, - // registeredChannel.joystreamChannelId - // ) - // } - // } - - // get user from userId - const user = await this.dynamodbService.users.get(id) - - // ensure request's authorization code matches the user's authorization code - if (user.youtubeVideoUrl !== youtubeVideoUrl) { - throw new Error('Authorization error. Youtube video url is invalid.') - } - - // Ensure that user has not already created a Joystream membership - if (user.joystreamMemberId) { - throw new Error(`Already created Joystream member ${user.joystreamMemberId} for user ${user.id}`) - } - - // send create membership request to faucet - const { memberId } = await this.createMemberWithFaucet({ account, handle, avatar, about, name }) - - // save updated user entity - await this.dynamodbService.users.save({ ...user, joystreamMemberId: memberId }) - - return new CreateMembershipResponse(memberId, handle) - } catch (error) { - const message = error instanceof Error ? error.message : error - throw new BadRequestException(message) - } - } - - private async createMemberWithFaucet(params: FaucetRegisterMembershipParams): Promise<{ memberId: number }> { - const { endpoint, captchaBypassKey } = this.config.joystream.faucet - try { - const response = await axios.post(endpoint, params, { - headers: { Authorization: `Bearer ${captchaBypassKey}` }, - }) - return response.data - } catch (error) { - if (axios.isAxiosError(error)) { - throw new FaucetApiError( - error.response?.data?.error || error.cause || error.code, - `Failed to create membership through faucet for account address: ${params.account}` - ) - } - throw error - } - } -} diff --git a/src/services/httpApi/controllers/users.ts b/src/services/httpApi/controllers/users.ts index 8985fdd9..09961ff5 100644 --- a/src/services/httpApi/controllers/users.ts +++ b/src/services/httpApi/controllers/users.ts @@ -48,12 +48,6 @@ export class UsersController { throw errors } - // Get existing user record from db (if any) - const existingUser = await this.dynamodbService.repo.users.get(user.id) - - // save user & set joystreamMemberId if user already existed - await this.dynamodbService.users.save({ ...user, joystreamMemberId: existingUser?.joystreamMemberId }) - // return verified user return { id: user.id, diff --git a/src/services/httpApi/dtos.ts b/src/services/httpApi/dtos.ts index 0eaf11ad..5efe1807 100644 --- a/src/services/httpApi/dtos.ts +++ b/src/services/httpApi/dtos.ts @@ -247,46 +247,6 @@ export class SaveChannelResponse { } } -// Dto for creating membership request -export class CreateMembershipRequest { - // ID of the verified Youtube channel - @IsString() @ApiProperty({ required: true }) id: string - - // Youtube video URL required for the verification - @IsUrl({ require_tld: false }) - @ApiProperty({ required: true }) - youtubeVideoUrl: string - - // Membership Account address - @IsString() @ApiProperty({ required: true }) account: string - - // Membership Handle - @IsString() @ApiProperty({ required: true }) handle: string - - // Membership avatar URL - @IsOptional() @IsUrl({ require_tld: false }) @ApiProperty({ required: true }) avatar: string - - // `about` information to associate with new Membership - @ApiProperty() about: string - - // Membership name - @ApiProperty() name: string -} - -// Dto for create membership response -export class CreateMembershipResponse { - // Membership Account address - @IsNumber() @ApiProperty({ required: true }) memberId: number - - // Membership Handle - @IsString() @ApiProperty({ required: true }) handle: string - - constructor(memberId: number, handle: string) { - this.memberId = memberId - this.handle = handle - } -} - export class VideoDto extends YtVideo { @ApiProperty() url: string @ApiProperty() title: string diff --git a/src/services/httpApi/main.ts b/src/services/httpApi/main.ts index f22652c3..256dabcb 100644 --- a/src/services/httpApi/main.ts +++ b/src/services/httpApi/main.ts @@ -17,7 +17,6 @@ import { ContentProcessingService } from '../syncProcessing' import { YoutubePollingService } from '../syncProcessing/YoutubePollingService' import { YoutubeApi } from '../youtube' import { ChannelsController, StatusController, UsersController, VideosController } from './controllers' -import { MembershipController } from './controllers/membership' import { ReferrersController } from './controllers/referrers' class ApiModule {} @@ -69,14 +68,7 @@ export async function bootstrapHttpApi( module: ApiModule, imports: [], exports: [], - controllers: [ - VideosController, - ChannelsController, - ReferrersController, - UsersController, - StatusController, - MembershipController, - ], + controllers: [VideosController, ChannelsController, ReferrersController, UsersController, StatusController], providers: [ { provide: DynamodbService, diff --git a/src/services/youtube/openApi.ts b/src/services/youtube/openApi.ts index 42aea778..8221e024 100644 --- a/src/services/youtube/openApi.ts +++ b/src/services/youtube/openApi.ts @@ -45,7 +45,6 @@ export class YtDlpClient implements IOpenYTApi { const user: YtUser = { id: video.channelId, - joystreamMemberId: undefined, youtubeVideoUrl: videoUrl, createdAt: new Date(), } diff --git a/src/types/errors.ts b/src/types/errors.ts index 7f0cfc3c..4bc85122 100644 --- a/src/types/errors.ts +++ b/src/types/errors.ts @@ -69,12 +69,3 @@ export class QueryNodeApiError { public expected?: number | string | Date ) {} } - -export class FaucetApiError { - constructor( - public code: string, - public message?: string, - public result?: number | string | Date, - public expected?: number | string | Date - ) {} -} diff --git a/src/types/generated/ConfigJson.d.ts b/src/types/generated/ConfigJson.d.ts index 61266bd5..e35fe179 100644 --- a/src/types/generated/ConfigJson.d.ts +++ b/src/types/generated/ConfigJson.d.ts @@ -13,19 +13,6 @@ export interface YoutubeSyncNodeConfiguration { * Joystream network related configuration */ joystream: { - /** - * Joystream's faucet configuration (needed for captcha-free membership creation) - */ - faucet: { - /** - * Joystream's faucet URL - */ - endpoint: string - /** - * Bearer Authentication Key needed to bypass captcha verification on Faucet - */ - captchaBypassKey: string - } /** * Joystream Metaprotocol App specific configuration */ diff --git a/src/types/youtube.ts b/src/types/youtube.ts index 011d7a5d..05a3bff2 100644 --- a/src/types/youtube.ts +++ b/src/types/youtube.ts @@ -163,9 +163,6 @@ export class YtUser { // The URL for a specific video of Youtube channel with which the user verified for YPP youtubeVideoUrl: string - // Corresponding Joystream member ID for Youtube user - joystreamMemberId: number | undefined - // Record created At timestamp createdAt: Date } @@ -223,9 +220,9 @@ export const videoStates = [...(Object.keys(VideoStates) as (keyof typeof VideoS export const channelYppStatus = readonlyChannelYppStatus as unknown as string[] -export type VideoState = typeof videoStates[number] +export type VideoState = (typeof videoStates)[number] -export type ChannelYppStatus = typeof readonlyChannelYppStatus[number] +export type ChannelYppStatus = (typeof readonlyChannelYppStatus)[number] export type JoystreamVideo = { // Joystream runtime Video ID for successfully synced video @@ -370,18 +367,6 @@ export type YtDlpVideoOutput = { }[] } -export type FaucetRegisterMembershipParams = { - account: string - handle: string - avatar: string - about: string - name: string -} - -export type FaucetRegisterMembershipResponse = { - memberId: number -} - export type ChannelSyncStatus = { backlogCount: number placeInSyncQueue: number From ad8b8c171481b80eff4585ac5c730ceb7bb7f822 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 6 Jun 2024 11:32:57 +0500 Subject: [PATCH 11/17] fix: regenerate docs --- ...ties-faucet-properties-captchabypasskey.md | 3 - ...m-properties-faucet-properties-endpoint.md | 3 - ...-properties-joystream-properties-faucet.md | 46 ------- ...auth2-client-configuration-dependencies.md | 3 - ...configuration-properties-adckeyfilepath.md | 3 - ...lient-configuration-properties-clientid.md | 3 - ...t-configuration-properties-clientsecret.md | 3 - ...erties-maxallowedquotausageinpercentage.md | 11 -- ...ies-youtube-oauth2-client-configuration.md | 92 ------------- ...ted-configuration-if-properties-apimode.md | 12 -- ...-configuration-if-properties-signupmode.md | 12 -- ...ube-related-configuration-if-properties.md | 3 - ...erties-youtube-related-configuration-if.md | 36 ----- ...elated-configuration-properties-apimode.md | 21 --- ...ted-configuration-properties-signupmode.md | 21 --- ...-youtube-api-configuration-dependencies.md | 3 - ...configuration-properties-adckeyfilepath.md | 3 - ...e-api-configuration-properties-clientid.md | 3 - ...i-configuration-properties-clientsecret.md | 3 - ...erties-maxallowedquotausageinpercentage.md | 11 -- ...on-properties-youtube-api-configuration.md | 92 ------------- ...ties-youtube-related-configuration-then.md | 3 - ...ding-related-configuration-dependencies.md | 3 - ...-configuration-if-properties-signupmode.md | 12 -- ...ing-related-configuration-if-properties.md | 3 - ...gnuponboarding-related-configuration-if.md | 36 ----- ...configuration-properties-adckeyfilepath.md | 3 - ...lated-configuration-properties-clientid.md | 3 - ...d-configuration-properties-clientsecret.md | 3 - ...erties-maxallowedquotausageinpercentage.md | 11 -- ...ted-configuration-properties-signupmode.md | 21 --- ...uponboarding-related-configuration-then.md | 3 - ...-signuponboarding-related-configuration.md | 129 ------------------ package.json | 2 +- 34 files changed, 1 insertion(+), 618 deletions(-) delete mode 100644 docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md delete mode 100644 docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md delete mode 100644 docs/config/definition-properties-joystream-properties-faucet.md delete mode 100644 docs/config/definition-properties-youtube-oauth2-client-configuration-dependencies.md delete mode 100644 docs/config/definition-properties-youtube-oauth2-client-configuration-properties-adckeyfilepath.md delete mode 100644 docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientid.md delete mode 100644 docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientsecret.md delete mode 100644 docs/config/definition-properties-youtube-oauth2-client-configuration-properties-maxallowedquotausageinpercentage.md delete mode 100644 docs/config/definition-properties-youtube-oauth2-client-configuration.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-if-properties.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-if.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-apimode.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md delete mode 100644 docs/config/definition-properties-youtube-related-configuration-then.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md delete mode 100644 docs/config/definition-properties-youtube-signuponboarding-related-configuration.md diff --git a/docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md b/docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md deleted file mode 100644 index c450d7b9..00000000 --- a/docs/config/definition-properties-joystream-properties-faucet-properties-captchabypasskey.md +++ /dev/null @@ -1,3 +0,0 @@ -## captchaBypassKey Type - -`string` diff --git a/docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md b/docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md deleted file mode 100644 index 00e8b7f7..00000000 --- a/docs/config/definition-properties-joystream-properties-faucet-properties-endpoint.md +++ /dev/null @@ -1,3 +0,0 @@ -## endpoint Type - -`string` diff --git a/docs/config/definition-properties-joystream-properties-faucet.md b/docs/config/definition-properties-joystream-properties-faucet.md deleted file mode 100644 index 3abaa799..00000000 --- a/docs/config/definition-properties-joystream-properties-faucet.md +++ /dev/null @@ -1,46 +0,0 @@ -## faucet Type - -`object` ([Details](definition-properties-joystream-properties-faucet.md)) - -# faucet Properties - -| Property | Type | Required | Nullable | Defined by | -| :------------------------------------ | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [endpoint](#endpoint) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-endpoint.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/endpoint") | -| [captchaBypassKey](#captchabypasskey) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-captchabypasskey.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/captchaBypassKey") | - -## endpoint - -Joystream's faucet URL - -`endpoint` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-endpoint.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/endpoint") - -### endpoint Type - -`string` - -## captchaBypassKey - -Bearer Authentication Key needed to bypass captcha verification on Faucet - -`captchaBypassKey` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-joystream-properties-faucet-properties-captchabypasskey.md "https://joystream.org/schemas/youtube-synch/config#/properties/joystream/properties/faucet/properties/captchaBypassKey") - -### captchaBypassKey Type - -`string` diff --git a/docs/config/definition-properties-youtube-oauth2-client-configuration-dependencies.md b/docs/config/definition-properties-youtube-oauth2-client-configuration-dependencies.md deleted file mode 100644 index 5fece311..00000000 --- a/docs/config/definition-properties-youtube-oauth2-client-configuration-dependencies.md +++ /dev/null @@ -1,3 +0,0 @@ -## dependencies Type - -unknown diff --git a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-adckeyfilepath.md b/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-adckeyfilepath.md deleted file mode 100644 index b029962a..00000000 --- a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-adckeyfilepath.md +++ /dev/null @@ -1,3 +0,0 @@ -## adcKeyFilePath Type - -`string` diff --git a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientid.md b/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientid.md deleted file mode 100644 index a7b5d5fb..00000000 --- a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientid.md +++ /dev/null @@ -1,3 +0,0 @@ -## clientId Type - -`string` diff --git a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientsecret.md b/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientsecret.md deleted file mode 100644 index 53452148..00000000 --- a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-clientsecret.md +++ /dev/null @@ -1,3 +0,0 @@ -## clientSecret Type - -`string` diff --git a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-maxallowedquotausageinpercentage.md b/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-maxallowedquotausageinpercentage.md deleted file mode 100644 index 7c692483..00000000 --- a/docs/config/definition-properties-youtube-oauth2-client-configuration-properties-maxallowedquotausageinpercentage.md +++ /dev/null @@ -1,11 +0,0 @@ -## maxAllowedQuotaUsageInPercentage Type - -`number` - -## maxAllowedQuotaUsageInPercentage Default Value - -The default value is: - -```json -95 -``` diff --git a/docs/config/definition-properties-youtube-oauth2-client-configuration.md b/docs/config/definition-properties-youtube-oauth2-client-configuration.md deleted file mode 100644 index e7b63522..00000000 --- a/docs/config/definition-properties-youtube-oauth2-client-configuration.md +++ /dev/null @@ -1,92 +0,0 @@ -## youtube Type - -`object` ([Youtube Oauth2 Client configuration](definition-properties-youtube-oauth2-client-configuration.md)) - -# youtube Properties - -| Property | Type | Required | Nullable | Defined by | -| :-------------------------------------------------------------------- | :------- | :------- | :------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [clientId](#clientid) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientId") | -| [clientSecret](#clientsecret) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientSecret") | -| [maxAllowedQuotaUsageInPercentage](#maxallowedquotausageinpercentage) | `number` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/maxAllowedQuotaUsageInPercentage") | -| [adcKeyFilePath](#adckeyfilepath) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/adcKeyFilePath") | - -## clientId - -Youtube Oauth2 Client Id - -`clientId` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientId") - -### clientId Type - -`string` - -## clientSecret - -Youtube Oauth2 Client Secret - -`clientSecret` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientSecret") - -### clientSecret Type - -`string` - -## maxAllowedQuotaUsageInPercentage - -Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups. - -`maxAllowedQuotaUsageInPercentage` - -* is optional - -* Type: `number` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/maxAllowedQuotaUsageInPercentage") - -### maxAllowedQuotaUsageInPercentage Type - -`number` - -### maxAllowedQuotaUsageInPercentage Default Value - -The default value is: - -```json -95 -``` - -## adcKeyFilePath - -Path to the Google Cloud's Application Default Credentials (ADC) key file. It is required to periodically monitor the Youtube API quota usage. - -`adcKeyFilePath` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-oauth2-client-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/adcKeyFilePath") - -### adcKeyFilePath Type - -`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md b/docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md deleted file mode 100644 index 646e3cc5..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-if-properties-apimode.md +++ /dev/null @@ -1,12 +0,0 @@ -## apiMode Type - -unknown - -## apiMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :------- | :---------- | -| `"api"` | | -| `"both"` | | diff --git a/docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md b/docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md deleted file mode 100644 index 4bd91053..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-if-properties-signupmode.md +++ /dev/null @@ -1,12 +0,0 @@ -## signupMode Type - -unknown - -## signupMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :------- | :---------- | -| `"api"` | | -| `"both"` | | diff --git a/docs/config/definition-properties-youtube-related-configuration-if-properties.md b/docs/config/definition-properties-youtube-related-configuration-if-properties.md deleted file mode 100644 index c89940c6..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-if-properties.md +++ /dev/null @@ -1,3 +0,0 @@ -## properties Type - -unknown diff --git a/docs/config/definition-properties-youtube-related-configuration-if.md b/docs/config/definition-properties-youtube-related-configuration-if.md deleted file mode 100644 index 00a9a02b..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-if.md +++ /dev/null @@ -1,36 +0,0 @@ -## if Type - -unknown - -# if Properties - -| Property | Type | Required | Nullable | Defined by | -| :------------------ | :------------ | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [apiMode](#apimode) | Not specified | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-if-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/apiMode") | - -## apiMode - - - -`apiMode` - -* is optional - -* Type: unknown - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-if-properties-apimode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/apiMode") - -### apiMode Type - -unknown - -### apiMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :------- | :---------- | -| `"api"` | | -| `"both"` | | diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-apimode.md b/docs/config/definition-properties-youtube-related-configuration-properties-apimode.md deleted file mode 100644 index 06e5445e..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-apimode.md +++ /dev/null @@ -1,21 +0,0 @@ -## apiMode Type - -`string` - -## apiMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :----------- | :---------- | -| `"api-free"` | | -| `"api"` | | -| `"both"` | | - -## apiMode Default Value - -The default value is: - -```json -"both" -``` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md b/docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md deleted file mode 100644 index c053e307..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-signupmode.md +++ /dev/null @@ -1,21 +0,0 @@ -## signupMode Type - -`string` - -## signupMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :----------- | :---------- | -| `"api-free"` | | -| `"api"` | | -| `"both"` | | - -## signupMode Default Value - -The default value is: - -```json -"both" -``` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md deleted file mode 100644 index 5fece311..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-dependencies.md +++ /dev/null @@ -1,3 +0,0 @@ -## dependencies Type - -unknown diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md deleted file mode 100644 index b029962a..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md +++ /dev/null @@ -1,3 +0,0 @@ -## adcKeyFilePath Type - -`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md deleted file mode 100644 index a7b5d5fb..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md +++ /dev/null @@ -1,3 +0,0 @@ -## clientId Type - -`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md deleted file mode 100644 index 53452148..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md +++ /dev/null @@ -1,3 +0,0 @@ -## clientSecret Type - -`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md deleted file mode 100644 index 7c692483..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md +++ /dev/null @@ -1,11 +0,0 @@ -## maxAllowedQuotaUsageInPercentage Type - -`number` - -## maxAllowedQuotaUsageInPercentage Default Value - -The default value is: - -```json -95 -``` diff --git a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md b/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md deleted file mode 100644 index 59a856b6..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md +++ /dev/null @@ -1,92 +0,0 @@ -## api Type - -`object` ([Youtube API configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration.md)) - -# api Properties - -| Property | Type | Required | Nullable | Defined by | -| :-------------------------------------------------------------------- | :------- | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [clientId](#clientid) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientId") | -| [clientSecret](#clientsecret) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientSecret") | -| [maxAllowedQuotaUsageInPercentage](#maxallowedquotausageinpercentage) | `number` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/maxAllowedQuotaUsageInPercentage") | -| [adcKeyFilePath](#adckeyfilepath) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/adcKeyFilePath") | - -## clientId - -Youtube Oauth2 Client Id - -`clientId` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientId") - -### clientId Type - -`string` - -## clientSecret - -Youtube Oauth2 Client Secret - -`clientSecret` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/clientSecret") - -### clientSecret Type - -`string` - -## maxAllowedQuotaUsageInPercentage - -Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups. - -`maxAllowedQuotaUsageInPercentage` - -* is optional - -* Type: `number` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/maxAllowedQuotaUsageInPercentage") - -### maxAllowedQuotaUsageInPercentage Type - -`number` - -### maxAllowedQuotaUsageInPercentage Default Value - -The default value is: - -```json -95 -``` - -## adcKeyFilePath - -Path to the Google Cloud's Application Default Credentials (ADC) key file. It is required to periodically monitor the Youtube API quota usage. - -`adcKeyFilePath` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-related-configuration-properties-youtube-api-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/api/properties/adcKeyFilePath") - -### adcKeyFilePath Type - -`string` diff --git a/docs/config/definition-properties-youtube-related-configuration-then.md b/docs/config/definition-properties-youtube-related-configuration-then.md deleted file mode 100644 index 139b21d9..00000000 --- a/docs/config/definition-properties-youtube-related-configuration-then.md +++ /dev/null @@ -1,3 +0,0 @@ -## then Type - -unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md deleted file mode 100644 index 5fece311..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-dependencies.md +++ /dev/null @@ -1,3 +0,0 @@ -## dependencies Type - -unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md deleted file mode 100644 index 4bd91053..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md +++ /dev/null @@ -1,12 +0,0 @@ -## signupMode Type - -unknown - -## signupMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :------- | :---------- | -| `"api"` | | -| `"both"` | | diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md deleted file mode 100644 index c89940c6..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if-properties.md +++ /dev/null @@ -1,3 +0,0 @@ -## properties Type - -unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md deleted file mode 100644 index 9af00676..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-if.md +++ /dev/null @@ -1,36 +0,0 @@ -## if Type - -unknown - -# if Properties - -| Property | Type | Required | Nullable | Defined by | -| :------------------------ | :------------ | :------- | :------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| [signupMode](#signupmode) | Not specified | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/signupMode") | - -## signupMode - - - -`signupMode` - -* is optional - -* Type: unknown - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-if-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/if/properties/signupMode") - -### signupMode Type - -unknown - -### signupMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :------- | :---------- | -| `"api"` | | -| `"both"` | | diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md deleted file mode 100644 index b029962a..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md +++ /dev/null @@ -1,3 +0,0 @@ -## adcKeyFilePath Type - -`string` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md deleted file mode 100644 index a7b5d5fb..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md +++ /dev/null @@ -1,3 +0,0 @@ -## clientId Type - -`string` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md deleted file mode 100644 index 53452148..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md +++ /dev/null @@ -1,3 +0,0 @@ -## clientSecret Type - -`string` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md deleted file mode 100644 index 7c692483..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md +++ /dev/null @@ -1,11 +0,0 @@ -## maxAllowedQuotaUsageInPercentage Type - -`number` - -## maxAllowedQuotaUsageInPercentage Default Value - -The default value is: - -```json -95 -``` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md deleted file mode 100644 index c053e307..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md +++ /dev/null @@ -1,21 +0,0 @@ -## signupMode Type - -`string` - -## signupMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :----------- | :---------- | -| `"api-free"` | | -| `"api"` | | -| `"both"` | | - -## signupMode Default Value - -The default value is: - -```json -"both" -``` diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md deleted file mode 100644 index 139b21d9..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration-then.md +++ /dev/null @@ -1,3 +0,0 @@ -## then Type - -unknown diff --git a/docs/config/definition-properties-youtube-signuponboarding-related-configuration.md b/docs/config/definition-properties-youtube-signuponboarding-related-configuration.md deleted file mode 100644 index 8f1778ce..00000000 --- a/docs/config/definition-properties-youtube-signuponboarding-related-configuration.md +++ /dev/null @@ -1,129 +0,0 @@ -## youtube Type - -`object` ([Youtube Signup/Onboarding related configuration](definition-properties-youtube-signuponboarding-related-configuration.md)) - -# youtube Properties - -| Property | Type | Required | Nullable | Defined by | -| :-------------------------------------------------------------------- | :------- | :------- | :------------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [signupMode](#signupmode) | `string` | Required | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/signupMode") | -| [clientId](#clientid) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientId") | -| [clientSecret](#clientsecret) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientSecret") | -| [maxAllowedQuotaUsageInPercentage](#maxallowedquotausageinpercentage) | `number` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/maxAllowedQuotaUsageInPercentage") | -| [adcKeyFilePath](#adckeyfilepath) | `string` | Optional | cannot be null | [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/adcKeyFilePath") | - -## signupMode - - - -`signupMode` - -* is required - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-signupmode.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/signupMode") - -### signupMode Type - -`string` - -### signupMode Constraints - -**enum**: the value of this property must be equal to one of the following values: - -| Value | Explanation | -| :----------- | :---------- | -| `"api-free"` | | -| `"api"` | | -| `"both"` | | - -### signupMode Default Value - -The default value is: - -```json -"both" -``` - -## clientId - -Youtube Oauth2 Client Id - -`clientId` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientid.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientId") - -### clientId Type - -`string` - -## clientSecret - -Youtube Oauth2 Client Secret - -`clientSecret` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-clientsecret.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/clientSecret") - -### clientSecret Type - -`string` - -## maxAllowedQuotaUsageInPercentage - -Maximum percentage of daily Youtube API quota that can be used by the Periodic polling service. Once this limit is reached the service will stop polling for new videos until the next day(when Quota resets). All the remaining quota (100 - maxAllowedQuotaUsageInPercentage) will be used for potential channel's signups. - -`maxAllowedQuotaUsageInPercentage` - -* is optional - -* Type: `number` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-maxallowedquotausageinpercentage.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/maxAllowedQuotaUsageInPercentage") - -### maxAllowedQuotaUsageInPercentage Type - -`number` - -### maxAllowedQuotaUsageInPercentage Default Value - -The default value is: - -```json -95 -``` - -## adcKeyFilePath - -Path to the Google Cloud's Application Default Credentials (ADC) key file. It is required to periodically monitor the Youtube API quota usage. - -`adcKeyFilePath` - -* is optional - -* Type: `string` - -* cannot be null - -* defined in: [Youtube Sync node configuration](definition-properties-youtube-signuponboarding-related-configuration-properties-adckeyfilepath.md "https://joystream.org/schemas/youtube-synch/config#/properties/youtube/properties/adcKeyFilePath") - -### adcKeyFilePath Type - -`string` diff --git a/package.json b/package.json index ae56fa86..61c35abf 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "dynamodb:local:start": "npm run dynamodb:local:stop && export DEPLOYMENT_ENV=local && ./src/infrastructure/deploy.sh", "dynamodb:local:stop": "export DEPLOYMENT_ENV=local REMOVE_PULUMI_STACK=true && ./src/infrastructure/destroy.sh", "generate:docs:cli": "npx oclif-dev readme --multi --dir ./docs/cli", - "generate:docs:config": "npx ts-node ./src/schemas/scripts/generateConfigDoc.ts", + "generate:docs:config": "rm -rf ./docs/config && npx ts-node ./src/schemas/scripts/generateConfigDoc.ts", "generate:docs:all": "npm run generate:docs:cli && npm run generate:docs:config", "generate:types:all": "npm run generate:types:json-schema && npm run generate:types:graphql", "generate:types:graphql": "npx graphql-codegen --config ./src/services/query-node/codegen.yml", From db12fb273af79307773a72e2182f5e8363a94e8c Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 6 Jun 2024 12:29:03 +0500 Subject: [PATCH 12/17] fix: youtube API --- src/services/httpApi/controllers/channels.ts | 2 +- .../syncProcessing/YoutubePollingService.ts | 5 +-- src/services/youtube/openApi.ts | 37 ++++++++++++++----- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts index caae2e82..cc219919 100644 --- a/src/services/httpApi/controllers/channels.ts +++ b/src/services/httpApi/controllers/channels.ts @@ -100,7 +100,7 @@ export class ChannelsController { let channel = await this.youtubeApi.operationalApi.getChannel(id) const existingChannel = await this.dynamodbService.repo.channels.get(channel.id) - const joystreamChannelLanguageIso = jsChannel.language?.iso + const joystreamChannelLanguageIso = jsChannel.language || undefined // If channel already exists in the DB (in `OptedOut` state), then we // associate most properties of existing channel record with the new diff --git a/src/services/syncProcessing/YoutubePollingService.ts b/src/services/syncProcessing/YoutubePollingService.ts index 81662c37..be962d75 100644 --- a/src/services/syncProcessing/YoutubePollingService.ts +++ b/src/services/syncProcessing/YoutubePollingService.ts @@ -169,10 +169,7 @@ export class YoutubePollingService { } // get all videos that are not yet being tracked - const untrackedVideos = await this.youtubeApi.ytdlp.getVideos( - channel, - untrackedVideosIds.map((v) => v.id) - ) + const untrackedVideos = await this.youtubeApi.ytdlp.getVideos(channel, untrackedVideosIds) // save all new videos to DB including await this.dynamodbService.repo.videos.upsertAll(untrackedVideos) diff --git a/src/services/youtube/openApi.ts b/src/services/youtube/openApi.ts index 8221e024..dd68340c 100644 --- a/src/services/youtube/openApi.ts +++ b/src/services/youtube/openApi.ts @@ -11,7 +11,7 @@ import { YtChannel, YtDlpFlatPlaylistOutput, YtDlpVideoOutput, YtUser, YtVideo } export interface IOpenYTApi { getChannel(channelId: string): Promise<{ id: string; title: string; description: string }> getVideoFromUrl(videoUrl: string): Promise - getVideos(channel: YtChannel, ids: string[]): Promise + getVideos(channel: YtChannel, ids: YtDlpFlatPlaylistOutput): Promise downloadVideo(videoUrl: string, outPath: string): ReturnType getUserAndVideoFromVideoUrl(videoUrl: string): Promise<{ user: YtUser; video: YtVideo }> } @@ -63,17 +63,30 @@ export class YtDlpClient implements IOpenYTApi { return response } - async getVideos(channel: YtChannel, ids: string[]): Promise { + async getVideos(channel: YtChannel, ids: YtDlpFlatPlaylistOutput): Promise { const videosMetadata: YtDlpVideoOutput[] = [] const idsChunks = _.chunk(ids, 50) for (const idsChunk of idsChunks) { - const videosMetadataChunk = await Promise.all( - idsChunk.map(async (id) => { - const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`) - return JSON.parse(stdout) as YtDlpVideoOutput - }) - ) + const videosMetadataChunk = ( + await Promise.all( + idsChunk.map(async ({ id, isShort }) => { + try { + const { stdout } = await this.exec(`${this.ytdlpPath} -J https://www.youtube.com/watch?v=${id}`) + return { ...JSON.parse(stdout), isShort } as YtDlpVideoOutput + } catch (err) { + if ( + err instanceof Error && + (err.message.includes(`This video is age-restricted`) || + err.message.includes(`Join this channel to get access to members-only content`)) + ) { + return + } + throw err + } + }) + ) + ).filter((v) => v) as YtDlpVideoOutput[] videosMetadata.push(...videosMetadataChunk) } @@ -102,6 +115,7 @@ export class YtDlpClient implements IOpenYTApi { license: video?.license === 'Creative Commons Attribution license (reuse allowed)' ? 'creativeCommon' : 'youtube', duration: video?.duration, container: video?.ext, + isShort: video.isShort, viewCount: video?.view_count || 0, state: 'New', } @@ -140,12 +154,17 @@ export class YtDlpClient implements IOpenYTApi { JSON.parse(stdout).entries.forEach((category: any) => { if (category.entries) { category.entries.forEach((video: any) => { - videos.push({ id: video.id, publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */ }) + videos.push({ + id: video.id, + publishedAt: new Date(video.timestamp * 1000) /** Convert UNIX to date */, + isShort: type === 'shorts', + }) }) } else { videos.push({ id: category.id, publishedAt: new Date(category.timestamp * 1000) /** Convert UNIX to date */, + isShort: type === 'shorts', }) } }) From 1234cbf7c08f8cc2d409d7d6cc1aff7c08cbf42d Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Thu, 6 Jun 2024 12:49:51 +0500 Subject: [PATCH 13/17] fix: docker compose file --- docker-compose.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4faa66f0..074fabc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,8 +28,7 @@ services: YT_SYNCH__ENDPOINTS__REDIS__PORT: ${YT_SYNCH__ENDPOINTS__REDIS__PORT} # YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT} # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY} - # YT_SYNCH__YOUTUBE__CLIENT_ID: ${YT_SYNCH__YOUTUBE__CLIENT_ID} - # YT_SYNCH__YOUTUBE__CLIENT_SECRET: ${YT_SYNCH__YOUTUBE__CLIENT_SECRET} + YT_SYNCH__YOUTUBE__OPERATIONAL_API__URL: http://youtube-operational-api # YT_SYNCH__AWS__ENDPOINT: http://host.docker.internal:4566 # YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID: ${YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID} # YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY: ${YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY} @@ -76,9 +75,7 @@ services: YT_SYNCH__ENDPOINTS__REDIS__PORT: ${YT_SYNCH__ENDPOINTS__REDIS__PORT} # YT_SYNCH__HTTP_API__PORT: ${YT_SYNCH__HTTP_API__PORT} # YT_SYNCH__HTTP_API__OWNER_KEY: ${YT_SYNCH__HTTP_API__OWNER_KEY} - YT_SYNCH__YOUTUBE__OPERATIONAL_API__URL: http://youtube-operational-api:8889 - # YT_SYNCH__YOUTUBE__API__CLIENT_ID: ${YT_SYNCH__YOUTUBE__API__CLIENT_ID} - # YT_SYNCH__YOUTUBE__API__CLIENT_SECRET: ${YT_SYNCH__YOUTUBE__API__CLIENT_SECRET} + YT_SYNCH__YOUTUBE__OPERATIONAL_API__URL: http://youtube-operational-api # YT_SYNCH__AWS__ENDPOINT: ${YT_SYNCH__AWS__ENDPOINT} # YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID: ${YT_SYNCH__AWS__CREDENTIALS__ACCESS_KEY_ID} # YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY: ${YT_SYNCH__AWS__CREDENTIALS__SECRET_ACCESS_KEY} @@ -86,8 +83,6 @@ services: # YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID: ${YT_SYNCH__JOYSTREAM__CHANNEL_COLLABORATOR__MEMBER_ID} # YT_SYNCH__JOYSTREAM__APP__NAME: ${YT_SYNCH__JOYSTREAM__APP__NAME} # YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED: ${YT_SYNCH__JOYSTREAM__APP__ACCOUNT_SEED} - ports: - - 127.0.0.1:${YT_SYNCH__HTTP_API__PORT}:${YT_SYNCH__HTTP_API__PORT} networks: - youtube-synch command: ['./bin/run', 'start', '--service', 'sync'] From a7a5fb9b90a396477f8999153910af8a1531c2e0 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 7 Jun 2024 13:31:48 +0500 Subject: [PATCH 14/17] fix: save YtUser to dynamodb in POST /user request --- src/services/httpApi/controllers/users.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/services/httpApi/controllers/users.ts b/src/services/httpApi/controllers/users.ts index 09961ff5..23b507d8 100644 --- a/src/services/httpApi/controllers/users.ts +++ b/src/services/httpApi/controllers/users.ts @@ -48,6 +48,9 @@ export class UsersController { throw errors } + // save user & set joystreamMemberId if user already existed + await this.dynamodbService.users.save(user) + // return verified user return { id: user.id, From 586fb49ec6eda314c89e363c37c13dd5f2090fa5 Mon Sep 17 00:00:00 2001 From: Zeeshan Akram <97m.zeeshan@gmail.com> Date: Fri, 7 Jun 2024 16:24:18 +0500 Subject: [PATCH 15/17] fix: regenerate graphql schema & queries --- src/services/httpApi/controllers/channels.ts | 3 +- src/services/query-node/generated/queries.ts | 16 +- src/services/query-node/generated/schema.ts | 3802 ++++++- .../query-node/queries/content.graphql | 4 +- .../query-node/queries/membership.graphql | 4 +- src/services/query-node/schema.graphql | 9304 ++++++++++++----- .../syncProcessing/ContentCreationService.ts | 2 +- 7 files changed, 10134 insertions(+), 3001 deletions(-) diff --git a/src/services/httpApi/controllers/channels.ts b/src/services/httpApi/controllers/channels.ts index cc219919..2d8eecb7 100644 --- a/src/services/httpApi/controllers/channels.ts +++ b/src/services/httpApi/controllers/channels.ts @@ -60,6 +60,7 @@ export class ChannelsController { const { id, youtubeVideoUrl, joystreamChannelId, shouldBeIngested, videoCategoryId, referrerChannelId } = channelInfo + // TODO: ensure video is still unlisted // TODO: check if needed. maybe add new flag `isSaved` to user record and check that instead // // Ensure that channel is not already registered for YPP program // if (await this.dynamodbService.repo.channels.get(id)) { @@ -458,7 +459,7 @@ export class ChannelsController { } // verify the message signature using Channel owner's address - const { isValid } = signatureVerify(JSON.stringify(message), signature, jsChannel.ownerMember.controllerAccount) + const { isValid } = signatureVerify(JSON.stringify(message), signature, jsChannel.ownerMember.controllerAccount.id) // Ensure that the signature is valid and the message is not a playback message if (!isValid || channel.lastActedAt >= message.timestamp) { diff --git a/src/services/query-node/generated/queries.ts b/src/services/query-node/generated/queries.ts index 2b922129..a65c35f6 100644 --- a/src/services/query-node/generated/queries.ts +++ b/src/services/query-node/generated/queries.ts @@ -69,7 +69,7 @@ export type ChannelFieldsFragment = { language?: string | null totalVideosCreated: number videos: Array<{ id: string; videoStateBloatBond: any }> - ownerMember?: { id: string; controllerAccount: string } | null + ownerMember?: { id: string; controllerAccount: { id: string } } | null } export type GetChannelByIdQueryVariables = Types.Exact<{ @@ -82,7 +82,7 @@ export type GetChannelByIdQuery = { language?: string | null totalVideosCreated: number videos: Array<{ id: string; videoStateBloatBond: any }> - ownerMember?: { id: string; controllerAccount: string } | null + ownerMember?: { id: string; controllerAccount: { id: string } } | null } | null } @@ -128,7 +128,7 @@ export type MemberMetadataFieldsFragment = { name?: string | null; about?: strin export type MembershipFieldsFragment = { id: string handle: string - controllerAccount: string + controllerAccount: { id: string } metadata?: { name?: string | null; about?: string | null } | null } @@ -140,7 +140,7 @@ export type GetMemberByIdQuery = { membershipByUniqueInput?: { id: string handle: string - controllerAccount: string + controllerAccount: { id: string } metadata?: { name?: string | null; about?: string | null } | null } | null } @@ -205,7 +205,9 @@ export const ChannelFields = gql` language ownerMember { id - controllerAccount + controllerAccount { + id + } } totalVideosCreated } @@ -238,7 +240,9 @@ export const MembershipFields = gql` fragment MembershipFields on Membership { id handle - controllerAccount + controllerAccount { + id + } metadata { ...MemberMetadataFields } diff --git a/src/services/query-node/generated/schema.ts b/src/services/query-node/generated/schema.ts index b87b2ae4..9694559d 100644 --- a/src/services/query-node/generated/schema.ts +++ b/src/services/query-node/generated/schema.ts @@ -26,18 +26,12 @@ export type Account = { id: Scalars['String']['output'] /** Indicates whether the access to the gateway account is blocked */ isBlocked: Scalars['Boolean']['output'] - /** Indicates whether the gateway account's e-mail has been confirmed or not. */ - isEmailConfirmed: Scalars['Boolean']['output'] /** Blockchain (joystream) account associated with the gateway account */ - joystreamAccount: Scalars['String']['output'] - /** On-chain membership associated with the gateway account */ - membership: Membership + joystreamAccount: BlockchainAccount /** notification preferences for the account */ notificationPreferences: AccountNotificationPreferences /** runtime notifications */ notifications: Array - /** ID of the channel which referred the user to the platform */ - referrerChannelId?: Maybe /** Time when the gateway account was registered */ registeredAt: Scalars['DateTime']['output'] /** The user associated with the gateway account (the Gateway Account Owner) */ @@ -56,9 +50,7 @@ export type AccountData = { email: Scalars['String']['output'] followedChannels: Array id: Scalars['String']['output'] - isEmailConfirmed: Scalars['Boolean']['output'] joystreamAccount: Scalars['String']['output'] - membershipId: Scalars['String']['output'] preferences?: Maybe } @@ -77,6 +69,15 @@ export type AccountNotificationPreferences = { channelPaymentReceived: NotificationPreference channelReceivedFundsFromWg: NotificationPreference creatorTimedAuctionExpired: NotificationPreference + crtIssued: NotificationPreference + crtMarketBurn: NotificationPreference + crtMarketMint: NotificationPreference + crtMarketStarted: NotificationPreference + crtRevenueShareEnded: NotificationPreference + crtRevenueSharePlanned: NotificationPreference + crtRevenueShareStarted: NotificationPreference + crtSaleMint: NotificationPreference + crtSaleStarted: NotificationPreference fundsFromCouncilReceived: NotificationPreference fundsFromWgReceived: NotificationPreference fundsToExternalWalletSent: NotificationPreference @@ -195,6 +196,24 @@ export type AccountNotificationPreferencesWhereInput = { channelReceivedFundsFromWg_isNull?: InputMaybe creatorTimedAuctionExpired?: InputMaybe creatorTimedAuctionExpired_isNull?: InputMaybe + crtIssued?: InputMaybe + crtIssued_isNull?: InputMaybe + crtMarketBurn?: InputMaybe + crtMarketBurn_isNull?: InputMaybe + crtMarketMint?: InputMaybe + crtMarketMint_isNull?: InputMaybe + crtMarketStarted?: InputMaybe + crtMarketStarted_isNull?: InputMaybe + crtRevenueShareEnded?: InputMaybe + crtRevenueShareEnded_isNull?: InputMaybe + crtRevenueSharePlanned?: InputMaybe + crtRevenueSharePlanned_isNull?: InputMaybe + crtRevenueShareStarted?: InputMaybe + crtRevenueShareStarted_isNull?: InputMaybe + crtSaleMint?: InputMaybe + crtSaleMint_isNull?: InputMaybe + crtSaleStarted?: InputMaybe + crtSaleStarted_isNull?: InputMaybe fundsFromCouncilReceived?: InputMaybe fundsFromCouncilReceived_isNull?: InputMaybe fundsFromWgReceived?: InputMaybe @@ -250,24 +269,8 @@ export enum AccountOrderByInput { IdDesc = 'id_DESC', IsBlockedAsc = 'isBlocked_ASC', IsBlockedDesc = 'isBlocked_DESC', - IsEmailConfirmedAsc = 'isEmailConfirmed_ASC', - IsEmailConfirmedDesc = 'isEmailConfirmed_DESC', - JoystreamAccountAsc = 'joystreamAccount_ASC', - JoystreamAccountDesc = 'joystreamAccount_DESC', - MembershipControllerAccountAsc = 'membership_controllerAccount_ASC', - MembershipControllerAccountDesc = 'membership_controllerAccount_DESC', - MembershipCreatedAtAsc = 'membership_createdAt_ASC', - MembershipCreatedAtDesc = 'membership_createdAt_DESC', - MembershipHandleRawAsc = 'membership_handleRaw_ASC', - MembershipHandleRawDesc = 'membership_handleRaw_DESC', - MembershipHandleAsc = 'membership_handle_ASC', - MembershipHandleDesc = 'membership_handle_DESC', - MembershipIdAsc = 'membership_id_ASC', - MembershipIdDesc = 'membership_id_DESC', - MembershipTotalChannelsCreatedAsc = 'membership_totalChannelsCreated_ASC', - MembershipTotalChannelsCreatedDesc = 'membership_totalChannelsCreated_DESC', - ReferrerChannelIdAsc = 'referrerChannelId_ASC', - ReferrerChannelIdDesc = 'referrerChannelId_DESC', + JoystreamAccountIdAsc = 'joystreamAccount_id_ASC', + JoystreamAccountIdDesc = 'joystreamAccount_id_DESC', RegisteredAtAsc = 'registeredAt_ASC', RegisteredAtDesc = 'registeredAt_DESC', UserIdAsc = 'user_id_ASC', @@ -316,50 +319,13 @@ export type AccountWhereInput = { isBlocked_eq?: InputMaybe isBlocked_isNull?: InputMaybe isBlocked_not_eq?: InputMaybe - isEmailConfirmed_eq?: InputMaybe - isEmailConfirmed_isNull?: InputMaybe - isEmailConfirmed_not_eq?: InputMaybe - joystreamAccount_contains?: InputMaybe - joystreamAccount_containsInsensitive?: InputMaybe - joystreamAccount_endsWith?: InputMaybe - joystreamAccount_eq?: InputMaybe - joystreamAccount_gt?: InputMaybe - joystreamAccount_gte?: InputMaybe - joystreamAccount_in?: InputMaybe> + joystreamAccount?: InputMaybe joystreamAccount_isNull?: InputMaybe - joystreamAccount_lt?: InputMaybe - joystreamAccount_lte?: InputMaybe - joystreamAccount_not_contains?: InputMaybe - joystreamAccount_not_containsInsensitive?: InputMaybe - joystreamAccount_not_endsWith?: InputMaybe - joystreamAccount_not_eq?: InputMaybe - joystreamAccount_not_in?: InputMaybe> - joystreamAccount_not_startsWith?: InputMaybe - joystreamAccount_startsWith?: InputMaybe - membership?: InputMaybe - membership_isNull?: InputMaybe notificationPreferences?: InputMaybe notificationPreferences_isNull?: InputMaybe notifications_every?: InputMaybe notifications_none?: InputMaybe notifications_some?: InputMaybe - referrerChannelId_contains?: InputMaybe - referrerChannelId_containsInsensitive?: InputMaybe - referrerChannelId_endsWith?: InputMaybe - referrerChannelId_eq?: InputMaybe - referrerChannelId_gt?: InputMaybe - referrerChannelId_gte?: InputMaybe - referrerChannelId_in?: InputMaybe> - referrerChannelId_isNull?: InputMaybe - referrerChannelId_lt?: InputMaybe - referrerChannelId_lte?: InputMaybe - referrerChannelId_not_contains?: InputMaybe - referrerChannelId_not_containsInsensitive?: InputMaybe - referrerChannelId_not_endsWith?: InputMaybe - referrerChannelId_not_eq?: InputMaybe - referrerChannelId_not_in?: InputMaybe> - referrerChannelId_not_startsWith?: InputMaybe - referrerChannelId_startsWith?: InputMaybe registeredAt_eq?: InputMaybe registeredAt_gt?: InputMaybe registeredAt_gte?: InputMaybe @@ -386,6 +352,295 @@ export type AddVideoViewResult = { viewsNum: Scalars['Int']['output'] } +export type AmmCurve = { + /** the amm intercept parameter b in the formula a * x + b */ + ammInitPrice: Scalars['BigInt']['output'] + /** the amm slope parameter a in the formula a * x + b */ + ammSlopeParameter: Scalars['BigInt']['output'] + /** quantity bought on the market by the amm */ + burnedByAmm: Scalars['BigInt']['output'] + /** finalized (i.e. closed) */ + finalized: Scalars['Boolean']['output'] + /** counter */ + id: Scalars['String']['output'] + /** quantity sold to the market */ + mintedByAmm: Scalars['BigInt']['output'] + /** token this Amm is for */ + token: CreatorToken + /** transaction for this amm */ + transactions: Array +} + +export type AmmCurveTransactionsArgs = { + limit?: InputMaybe + offset?: InputMaybe + orderBy?: InputMaybe> + where?: InputMaybe +} + +export type AmmCurveEdge = { + cursor: Scalars['String']['output'] + node: AmmCurve +} + +export enum AmmCurveOrderByInput { + AmmInitPriceAsc = 'ammInitPrice_ASC', + AmmInitPriceDesc = 'ammInitPrice_DESC', + AmmSlopeParameterAsc = 'ammSlopeParameter_ASC', + AmmSlopeParameterDesc = 'ammSlopeParameter_DESC', + BurnedByAmmAsc = 'burnedByAmm_ASC', + BurnedByAmmDesc = 'burnedByAmm_DESC', + FinalizedAsc = 'finalized_ASC', + FinalizedDesc = 'finalized_DESC', + IdAsc = 'id_ASC', + IdDesc = 'id_DESC', + MintedByAmmAsc = 'mintedByAmm_ASC', + MintedByAmmDesc = 'mintedByAmm_DESC', + TokenAccountsNumAsc = 'token_accountsNum_ASC', + TokenAccountsNumDesc = 'token_accountsNum_DESC', + TokenAnnualCreatorRewardPermillAsc = 'token_annualCreatorRewardPermill_ASC', + TokenAnnualCreatorRewardPermillDesc = 'token_annualCreatorRewardPermill_DESC', + TokenCreatedAtAsc = 'token_createdAt_ASC', + TokenCreatedAtDesc = 'token_createdAt_DESC', + TokenDeissuedAsc = 'token_deissued_ASC', + TokenDeissuedDesc = 'token_deissued_DESC', + TokenDescriptionAsc = 'token_description_ASC', + TokenDescriptionDesc = 'token_description_DESC', + TokenIdAsc = 'token_id_ASC', + TokenIdDesc = 'token_id_DESC', + TokenIsFeaturedAsc = 'token_isFeatured_ASC', + TokenIsFeaturedDesc = 'token_isFeatured_DESC', + TokenIsInviteOnlyAsc = 'token_isInviteOnly_ASC', + TokenIsInviteOnlyDesc = 'token_isInviteOnly_DESC', + TokenLastPriceAsc = 'token_lastPrice_ASC', + TokenLastPriceDesc = 'token_lastPrice_DESC', + TokenNumberOfRevenueShareActivationsAsc = 'token_numberOfRevenueShareActivations_ASC', + TokenNumberOfRevenueShareActivationsDesc = 'token_numberOfRevenueShareActivations_DESC', + TokenNumberOfVestedTransferIssuedAsc = 'token_numberOfVestedTransferIssued_ASC', + TokenNumberOfVestedTransferIssuedDesc = 'token_numberOfVestedTransferIssued_DESC', + TokenRevenueShareRatioPermillAsc = 'token_revenueShareRatioPermill_ASC', + TokenRevenueShareRatioPermillDesc = 'token_revenueShareRatioPermill_DESC', + TokenStatusAsc = 'token_status_ASC', + TokenStatusDesc = 'token_status_DESC', + TokenSymbolAsc = 'token_symbol_ASC', + TokenSymbolDesc = 'token_symbol_DESC', + TokenTotalSupplyAsc = 'token_totalSupply_ASC', + TokenTotalSupplyDesc = 'token_totalSupply_DESC', + TokenWhitelistApplicantLinkAsc = 'token_whitelistApplicantLink_ASC', + TokenWhitelistApplicantLinkDesc = 'token_whitelistApplicantLink_DESC', + TokenWhitelistApplicantNoteAsc = 'token_whitelistApplicantNote_ASC', + TokenWhitelistApplicantNoteDesc = 'token_whitelistApplicantNote_DESC', +} + +export type AmmCurveWhereInput = { + AND?: InputMaybe> + OR?: InputMaybe> + ammInitPrice_eq?: InputMaybe + ammInitPrice_gt?: InputMaybe + ammInitPrice_gte?: InputMaybe + ammInitPrice_in?: InputMaybe> + ammInitPrice_isNull?: InputMaybe + ammInitPrice_lt?: InputMaybe + ammInitPrice_lte?: InputMaybe + ammInitPrice_not_eq?: InputMaybe + ammInitPrice_not_in?: InputMaybe> + ammSlopeParameter_eq?: InputMaybe + ammSlopeParameter_gt?: InputMaybe + ammSlopeParameter_gte?: InputMaybe + ammSlopeParameter_in?: InputMaybe> + ammSlopeParameter_isNull?: InputMaybe + ammSlopeParameter_lt?: InputMaybe + ammSlopeParameter_lte?: InputMaybe + ammSlopeParameter_not_eq?: InputMaybe + ammSlopeParameter_not_in?: InputMaybe> + burnedByAmm_eq?: InputMaybe + burnedByAmm_gt?: InputMaybe + burnedByAmm_gte?: InputMaybe + burnedByAmm_in?: InputMaybe> + burnedByAmm_isNull?: InputMaybe + burnedByAmm_lt?: InputMaybe + burnedByAmm_lte?: InputMaybe + burnedByAmm_not_eq?: InputMaybe + burnedByAmm_not_in?: InputMaybe> + finalized_eq?: InputMaybe + finalized_isNull?: InputMaybe + finalized_not_eq?: InputMaybe + id_contains?: InputMaybe + id_containsInsensitive?: InputMaybe + id_endsWith?: InputMaybe + id_eq?: InputMaybe + id_gt?: InputMaybe + id_gte?: InputMaybe + id_in?: InputMaybe> + id_isNull?: InputMaybe + id_lt?: InputMaybe + id_lte?: InputMaybe + id_not_contains?: InputMaybe + id_not_containsInsensitive?: InputMaybe + id_not_endsWith?: InputMaybe + id_not_eq?: InputMaybe + id_not_in?: InputMaybe> + id_not_startsWith?: InputMaybe + id_startsWith?: InputMaybe + mintedByAmm_eq?: InputMaybe + mintedByAmm_gt?: InputMaybe + mintedByAmm_gte?: InputMaybe + mintedByAmm_in?: InputMaybe> + mintedByAmm_isNull?: InputMaybe + mintedByAmm_lt?: InputMaybe + mintedByAmm_lte?: InputMaybe + mintedByAmm_not_eq?: InputMaybe + mintedByAmm_not_in?: InputMaybe> + token?: InputMaybe + token_isNull?: InputMaybe + transactions_every?: InputMaybe + transactions_none?: InputMaybe + transactions_some?: InputMaybe +} + +export type AmmCurvesConnection = { + edges: Array + pageInfo: PageInfo + totalCount: Scalars['Int']['output'] +} + +export type AmmTransaction = { + /** buyer account */ + account: TokenAccount + /** Reference to the Amm Sale */ + amm: AmmCurve + /** block */ + createdIn: Scalars['Int']['output'] + /** counter */ + id: Scalars['String']['output'] + /** total HAPI paid/received for the quantity */ + pricePaid: Scalars['BigInt']['output'] + /** price per unit in HAPI */ + pricePerUnit: Scalars['BigInt']['output'] + /** amount of token bought/sold */ + quantity: Scalars['BigInt']['output'] + /** was it bought/sold */ + transactionType: AmmTransactionType +} + +export type AmmTransactionEdge = { + cursor: Scalars['String']['output'] + node: AmmTransaction +} + +export enum AmmTransactionOrderByInput { + AccountDeletedAsc = 'account_deleted_ASC', + AccountDeletedDesc = 'account_deleted_DESC', + AccountIdAsc = 'account_id_ASC', + AccountIdDesc = 'account_id_DESC', + AccountStakedAmountAsc = 'account_stakedAmount_ASC', + AccountStakedAmountDesc = 'account_stakedAmount_DESC', + AccountTotalAmountAsc = 'account_totalAmount_ASC', + AccountTotalAmountDesc = 'account_totalAmount_DESC', + AmmAmmInitPriceAsc = 'amm_ammInitPrice_ASC', + AmmAmmInitPriceDesc = 'amm_ammInitPrice_DESC', + AmmAmmSlopeParameterAsc = 'amm_ammSlopeParameter_ASC', + AmmAmmSlopeParameterDesc = 'amm_ammSlopeParameter_DESC', + AmmBurnedByAmmAsc = 'amm_burnedByAmm_ASC', + AmmBurnedByAmmDesc = 'amm_burnedByAmm_DESC', + AmmFinalizedAsc = 'amm_finalized_ASC', + AmmFinalizedDesc = 'amm_finalized_DESC', + AmmIdAsc = 'amm_id_ASC', + AmmIdDesc = 'amm_id_DESC', + AmmMintedByAmmAsc = 'amm_mintedByAmm_ASC', + AmmMintedByAmmDesc = 'amm_mintedByAmm_DESC', + CreatedInAsc = 'createdIn_ASC', + CreatedInDesc = 'createdIn_DESC', + IdAsc = 'id_ASC', + IdDesc = 'id_DESC', + PricePaidAsc = 'pricePaid_ASC', + PricePaidDesc = 'pricePaid_DESC', + PricePerUnitAsc = 'pricePerUnit_ASC', + PricePerUnitDesc = 'pricePerUnit_DESC', + QuantityAsc = 'quantity_ASC', + QuantityDesc = 'quantity_DESC', + TransactionTypeAsc = 'transactionType_ASC', + TransactionTypeDesc = 'transactionType_DESC', +} + +export enum AmmTransactionType { + Buy = 'BUY', + Sell = 'SELL', +} + +export type AmmTransactionWhereInput = { + AND?: InputMaybe> + OR?: InputMaybe> + account?: InputMaybe + account_isNull?: InputMaybe + amm?: InputMaybe + amm_isNull?: InputMaybe + createdIn_eq?: InputMaybe + createdIn_gt?: InputMaybe + createdIn_gte?: InputMaybe + createdIn_in?: InputMaybe> + createdIn_isNull?: InputMaybe + createdIn_lt?: InputMaybe + createdIn_lte?: InputMaybe + createdIn_not_eq?: InputMaybe + createdIn_not_in?: InputMaybe> + id_contains?: InputMaybe + id_containsInsensitive?: InputMaybe + id_endsWith?: InputMaybe + id_eq?: InputMaybe + id_gt?: InputMaybe + id_gte?: InputMaybe + id_in?: InputMaybe> + id_isNull?: InputMaybe + id_lt?: InputMaybe + id_lte?: InputMaybe + id_not_contains?: InputMaybe + id_not_containsInsensitive?: InputMaybe + id_not_endsWith?: InputMaybe + id_not_eq?: InputMaybe + id_not_in?: InputMaybe> + id_not_startsWith?: InputMaybe + id_startsWith?: InputMaybe + pricePaid_eq?: InputMaybe + pricePaid_gt?: InputMaybe + pricePaid_gte?: InputMaybe + pricePaid_in?: InputMaybe> + pricePaid_isNull?: InputMaybe + pricePaid_lt?: InputMaybe + pricePaid_lte?: InputMaybe + pricePaid_not_eq?: InputMaybe + pricePaid_not_in?: InputMaybe> + pricePerUnit_eq?: InputMaybe + pricePerUnit_gt?: InputMaybe + pricePerUnit_gte?: InputMaybe + pricePerUnit_in?: InputMaybe> + pricePerUnit_isNull?: InputMaybe + pricePerUnit_lt?: InputMaybe + pricePerUnit_lte?: InputMaybe + pricePerUnit_not_eq?: InputMaybe + pricePerUnit_not_in?: InputMaybe> + quantity_eq?: InputMaybe + quantity_gt?: InputMaybe + quantity_gte?: InputMaybe + quantity_in?: InputMaybe> + quantity_isNull?: InputMaybe + quantity_lt?: InputMaybe + quantity_lte?: InputMaybe + quantity_not_eq?: InputMaybe + quantity_not_in?: InputMaybe> + transactionType_eq?: InputMaybe + transactionType_in?: InputMaybe> + transactionType_isNull?: InputMaybe + transactionType_not_eq?: InputMaybe + transactionType_not_in?: InputMaybe> +} + +export type AmmTransactionsConnection = { + edges: Array + pageInfo: PageInfo + totalCount: Scalars['Int']['output'] +} + export type App = { appChannels: Array appVideos: Array