Merge pull request #5191 from Lezek123/colossus-cleanup-fix

Colossus cleanup fix

docker-compose.yml

@@ -105,6 +105,9 @@ services:
     environment:
       # ACCOUNT_URI overrides command line arg --accountUri
       - ACCOUNT_URI=${COLOSSUS_2_TRANSACTOR_URI}
+      # Env that allows testing cleanup
+      - CLEANUP_NEW_OBJECT_EXPIRATION_PERIOD=10
+      - CLEANUP_MIN_REPLICATION_THRESHOLD=1
     entrypoint: ['yarn', 'storage-node']
     command: [
       'server', '--worker=${COLOSSUS_2_WORKER_ID}', '--port=3333', '--uploads=/data/uploads',
@@ -113,7 +116,10 @@ services:
       '--apiUrl=${JOYSTREAM_NODE_WS}',
       '--logFilePath=/logs',
       '--tempFolder=/data/temp/',
-      '--pendingFolder=/data/pending/'
+      '--pendingFolder=/data/pending/',
+      # Use cleanup on colossus-2 for testing purposes
+      '--cleanup',
+      '--cleanupInterval=1'
     ]
 
   distributor-2:

storage-node/CHANGELOG.md

@@ -1,3 +1,10 @@
+### 4.2.0
+
+- Fix `util:cleanup` script (call `loadDataObjectIdCache` first)
+- Allow changing default log level using environment variable (`COLOSSUS_DEFAULT_LOG_LEVEL`)
+- Allow adjusting cleanup constants via env (`CLEANUP_MIN_REPLICATION_THRESHOLD`, `CLEANUP_NEW_OBJECT_EXPIRATION_PERIOD`)
+- Error handling: Clearer warning messages if unexpected response encountered during sync (ie. 404)
+
 ### 4.1.2
 
 - Bump @joystream/types to Petra version

storage-node/README.md

@@ -213,7 +213,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/cancel-invite.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/cancel-invite.ts)_
+_See code: [src/commands/leader/cancel-invite.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/cancel-invite.ts)_
 
 ## `storage-node leader:create-bucket`
 
@@ -244,7 +244,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/create-bucket.ts)_
+_See code: [src/commands/leader/create-bucket.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/create-bucket.ts)_
 
 ## `storage-node leader:delete-bucket`
 
@@ -271,7 +271,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/delete-bucket.ts)_
+_See code: [src/commands/leader/delete-bucket.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/delete-bucket.ts)_
 
 ## `storage-node leader:invite-operator`
 
@@ -300,7 +300,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/invite-operator.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/invite-operator.ts)_
+_See code: [src/commands/leader/invite-operator.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/invite-operator.ts)_
 
 ## `storage-node leader:remove-operator`
 
@@ -327,7 +327,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/remove-operator.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/remove-operator.ts)_
+_See code: [src/commands/leader/remove-operator.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/remove-operator.ts)_
 
 ## `storage-node leader:set-bucket-limits`
 
@@ -357,7 +357,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/set-bucket-limits.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/set-bucket-limits.ts)_
+_See code: [src/commands/leader/set-bucket-limits.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/set-bucket-limits.ts)_
 
 ## `storage-node leader:set-global-uploading-status`
 
@@ -385,7 +385,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/set-global-uploading-status.ts)_
+_See code: [src/commands/leader/set-global-uploading-status.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/set-global-uploading-status.ts)_
 
 ## `storage-node leader:update-bag-limit`
 
@@ -412,7 +412,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-bag-limit.ts)_
+_See code: [src/commands/leader/update-bag-limit.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-bag-limit.ts)_
 
 ## `storage-node leader:update-bags`
 
@@ -468,7 +468,7 @@ OPTIONS
       Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-bags.ts)_
+_See code: [src/commands/leader/update-bags.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-bags.ts)_
 
 ## `storage-node leader:update-blacklist`
 
@@ -497,7 +497,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-blacklist.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-blacklist.ts)_
+_See code: [src/commands/leader/update-blacklist.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-blacklist.ts)_
 
 ## `storage-node leader:update-bucket-status`
 
@@ -526,7 +526,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-bucket-status.ts)_
+_See code: [src/commands/leader/update-bucket-status.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-bucket-status.ts)_
 
 ## `storage-node leader:update-data-fee`
 
@@ -553,7 +553,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-data-fee.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-data-fee.ts)_
+_See code: [src/commands/leader/update-data-fee.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-data-fee.ts)_
 
 ## `storage-node leader:update-data-object-bloat-bond`
 
@@ -581,7 +581,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-data-object-bloat-bond.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-data-object-bloat-bond.ts)_
+_See code: [src/commands/leader/update-data-object-bloat-bond.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-data-object-bloat-bond.ts)_
 
 ## `storage-node leader:update-dynamic-bag-policy`
 
@@ -611,7 +611,7 @@ OPTIONS
   --keyStore=keyStore             Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-dynamic-bag-policy.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-dynamic-bag-policy.ts)_
+_See code: [src/commands/leader/update-dynamic-bag-policy.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-dynamic-bag-policy.ts)_
 
 ## `storage-node leader:update-voucher-limits`
 
@@ -640,7 +640,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/leader/update-voucher-limits.ts)_
+_See code: [src/commands/leader/update-voucher-limits.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/leader/update-voucher-limits.ts)_
 
 ## `storage-node operator:accept-invitation`
 
@@ -673,7 +673,7 @@ OPTIONS
   --keyStore=keyStore                            Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/operator/accept-invitation.ts)_
+_See code: [src/commands/operator/accept-invitation.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/operator/accept-invitation.ts)_
 
 ## `storage-node operator:set-metadata`
 
@@ -704,7 +704,7 @@ OPTIONS
   --keyStore=keyStore          Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/operator/set-metadata.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/operator/set-metadata.ts)_
+_See code: [src/commands/operator/set-metadata.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/operator/set-metadata.ts)_
 
 ## `storage-node server`
 
@@ -810,7 +810,7 @@ OPTIONS
                                                                    directory will be used.
 ```
 
-_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/server.ts)_
+_See code: [src/commands/server.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/server.ts)_
 
 ## `storage-node util:cleanup`
 
@@ -848,7 +848,7 @@ OPTIONS
   --keyStore=keyStore                              Path to a folder with multiple key files to load into keystore.
 ```
 
-_See code: [src/commands/util/cleanup.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/util/cleanup.ts)_
+_See code: [src/commands/util/cleanup.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/util/cleanup.ts)_
 
 ## `storage-node util:fetch-bucket`
 
@@ -881,7 +881,7 @@ OPTIONS
                                                      under the uploads directory will be used.
 ```
 
-_See code: [src/commands/util/fetch-bucket.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/util/fetch-bucket.ts)_
+_See code: [src/commands/util/fetch-bucket.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/util/fetch-bucket.ts)_
 
 ## `storage-node util:multihash`
 
@@ -896,7 +896,7 @@ OPTIONS
   -h, --help       show CLI help
 ```
 
-_See code: [src/commands/util/multihash.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/util/multihash.ts)_
+_See code: [src/commands/util/multihash.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/util/multihash.ts)_
 
 ## `storage-node util:verify-bag-id`
 
@@ -924,5 +924,5 @@ OPTIONS
       - dynamic:member:4
 ```
 
-_See code: [src/commands/util/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v4.0.0/src/commands/util/verify-bag-id.ts)_
+_See code: [src/commands/util/verify-bag-id.ts](https://github.com/Joystream/joystream/blob/v4.2.0/src/commands/util/verify-bag-id.ts)_
 <!-- commandsstop -->

storage-node/package.json

@@ -1,7 +1,7 @@
 {
   "name": "storage-node",
   "description": "Joystream storage subsystem.",
-  "version": "4.1.2",
+  "version": "4.2.0",
   "author": "Joystream contributors",
   "bin": {
     "storage-node": "./bin/run"

storage-node/src/commands/util/cleanup.ts

@@ -4,6 +4,7 @@ import ApiCommandBase from '../../command-base/ApiCommandBase'
 import logger from '../../services/logger'
 import { QueryNodeApi } from '../../services/queryNode/api'
 import { performCleanup } from '../../services/sync/cleanupService'
+import { loadDataObjectIdCache } from '../../services/caching/localDataObjects'
 
 /**
  * CLI command:
@@ -51,6 +52,7 @@ export default class Cleanup extends ApiCommandBase {
     const bucketId = flags.bucketId.toString()
     const api = await this.getApi()
     const qnApi = new QueryNodeApi(flags.queryNodeEndpoint)
+    await loadDataObjectIdCache(flags.uploads)
 
     logger.info('Cleanup...')
 

storage-node/src/services/caching/newUploads.ts

@@ -1,7 +1,8 @@
 import NodeCache from 'node-cache'
 
 // Expiration period in seconds for the new uploads data.
-const ExpirationPeriod = 3600 // seconds (1 hour)
+// Default=3600 (1 hour)
+const ExpirationPeriod = parseInt(process.env.CLEANUP_NEW_OBJECT_EXPIRATION_PERIOD || '0') || 3600
 
 // Max ID number in local cache
 const MaxEntries = 100000

storage-node/src/services/logger.ts

@@ -24,6 +24,10 @@ const levels = {
  */
 function createDefaultLoggerOptions(): winston.LoggerOptions {
   const level = () => {
+    const levelFlag = process.env.COLOSSUS_DEFAULT_LOG_LEVEL
+    if (levelFlag && Object.keys(levels).includes(levelFlag)) {
+      return levelFlag
+    }
     const env = process.env.NODE_ENV || 'development'
     const isDevelopment = env === 'development'
     return isDevelopment ? 'debug' : 'warn'

storage-node/src/services/sync/cleanupService.ts

@@ -20,8 +20,9 @@ export const MAXIMUM_QN_LAGGING_THRESHOLD = 100
 /**
  * The number of (peer) storage operators that should hold the assets, before the
  * cleanup/pruning of the outdated assets from this storage node can be initialed.
+ * Default=2.
  */
-export const MINIMUM_REPLICATION_THRESHOLD = 2
+export const MINIMUM_REPLICATION_THRESHOLD = parseInt(process.env.CLEANUP_MIN_REPLICATION_THRESHOLD || '0') || 2
 
 /**
  * Runs the data objects cleanup/pruning workflow. It removes all the local
@@ -80,6 +81,7 @@ export async function performCleanup(
   const removedIds = _.difference(storedObjectsIds, assignedObjectsIds)
   const removedObjects = await getDataObjectsByIDs(qnApi, removedIds)
 
+  logger.debug(`Cleanup - stored objects: ${storedObjectsIds.length}, assigned objects: ${assignedObjectsIds.length}`)
   logger.debug(`Cleanup - pruning ${removedIds.length} obsolete objects`)
 
   // Data objects permanently deleted from the runtime

storage-node/src/services/sync/tasks.ts

@@ -2,7 +2,7 @@ import fs from 'fs'
 import _ from 'lodash'
 import path from 'path'
 import { pipeline } from 'stream'
-import superagent from 'superagent'
+import superagent, { Response } from 'superagent'
 import urljoin from 'url-join'
 import { promisify } from 'util'
 import { v4 as uuidv4 } from 'uuid'
@@ -138,9 +138,9 @@ export class DownloadFileTask implements SyncTask {
         .set('X-COLOSSUS-HOST-ID', this.hostId) as unknown as NodeJS.ReadableStream
       const fileStream = fs.createWriteStream(tempFilePath)
 
-      request.on('response', (res) => {
-        if (!res.ok && res.statusCode !== 404) {
-          logger.warn(`Sync - unexpected status code(${res.statusCode}) for ${res?.request?.url}`)
+      request.on('response', (res: Response) => {
+        if (!res.ok) {
+          request.emit('error', `request failed: ${res.error}`)
         }
 
         // Handle 'error' event on Response too, because it will be emitted if request was
@@ -151,9 +151,6 @@ export class DownloadFileTask implements SyncTask {
         })
       })
 
-      request.on('error', (err) => {
-        logger.warn(`Sync - fetching data error for ${url}: ${err}`, { err })
-      })
       await streamPipeline(request, fileStream)
       await this.verifyDownloadedFile(tempFilePath)
       await moveFile(tempFilePath, filepath)

tests/network-tests/ColossusApi.ts

@@ -1,15 +1,17 @@
-import { Configuration, FilesApi } from '@joystream/storage-node-client'
+import { Configuration, FilesApi, StateApi } from '@joystream/storage-node-client'
 import { AxiosResponse } from 'axios'
 import { assert } from 'chai'
 
 export class ColossusApi {
   public filesApi: FilesApi
+  public stateApi: StateApi
 
   constructor(url: string) {
     const config = new Configuration({
       basePath: url,
     })
     this.filesApi = new FilesApi(config)
+    this.stateApi = new StateApi(config)
   }
 
   public async fetchAssetAsBuffer(assetId: string): Promise<AxiosResponse<Buffer>> {
@@ -28,4 +30,8 @@ export class ColossusApi {
     assert.deepEqual([...response.data], [...expectedData])
     assert.equal(response.headers['content-type'], expectedMimeType)
   }
+
+  public async expectAssetNotFound(assetId: string): Promise<AxiosResponse<unknown>> {
+    return this.filesApi.filesApiGetFile(assetId, { validateStatus: (status) => status === 404 })
+  }
 }

tests/network-tests/src/Api.ts

@@ -652,7 +652,7 @@ export class Api {
   public async untilProposalsCanBeCreated(
     numberOfProposals = 1,
     intervalMs = BLOCKTIME,
-    timeoutMs = 180000
+    maxRetries = 180
   ): Promise<void> {
     await Utils.until(
       `${numberOfProposals} proposals can be created`,
@@ -663,17 +663,17 @@ export class Api {
         return maxActiveProposalLimit.sub(activeProposalsN).toNumber() >= numberOfProposals
       },
       intervalMs,
-      timeoutMs
+      maxRetries
     )
   }
 
   public async untilCouncilStage(
     targetStage: 'Announcing' | 'Voting' | 'Revealing' | 'Idle',
     announcementPeriodNr: number | null = null,
     blocksReserve = 4,
-    intervalMs = BLOCKTIME
+    intervalMs = BLOCKTIME,
+    maxRetries = 600
   ): Promise<void> {
-    const stageTimeoutMs = 100 * 6 * 1000
     await Utils.until(
       `council stage ${targetStage} (+${blocksReserve} blocks reserve)`,
       async ({ debug }) => {
@@ -710,7 +710,7 @@ export class Api {
         )
       },
       intervalMs,
-      stageTimeoutMs
+      maxRetries
     )
   }