Skip to content

Commit

Permalink
refactor removeObjects api to ts
Browse files Browse the repository at this point in the history
  • Loading branch information
prakashsvmx committed May 3, 2024
1 parent e61a9e2 commit 5ef8ae1
Show file tree
Hide file tree
Showing 10 changed files with 155 additions and 206 deletions.
44 changes: 17 additions & 27 deletions docs/API.md
Original file line number Diff line number Diff line change
Expand Up @@ -1158,18 +1158,17 @@ console.log(stat)

<a name="removeObject"></a>

### removeObject(bucketName, objectName [, removeOpts] [, callback])
### removeObject(bucketName, objectName [, removeOpts])

Removes an object.

**Parameters**

| Param | Type | Description |
| --------------- | ---------- | ----------------------------------------------------------------------------------------------------------------------------- |
| `bucketName` | _string_ | Name of the bucket. |
| `objectName` | _string_ | Name of the object. |
| `removeOpts` | _object_ | Version of the object in the form `{versionId:"my-versionId", governanceBypass: true or false }`. Default is `{}`. (Optional) |
| `callback(err)` | _function_ | Callback function is called with non `null` value in case of error. If no callback is passed, a `Promise` is returned. |
| Param | Type | Description |
| ------------ | -------- | ----------------------------------------------------------------------------------------------------------------------------- |
| `bucketName` | _string_ | Name of the bucket. |
| `objectName` | _string_ | Name of the object. |
| `removeOpts` | _object_ | Version of the object in the form `{versionId:"my-versionId", governanceBypass: true or false }`. Default is `{}`. (Optional) |

**Example 1**

Expand Down Expand Up @@ -1206,17 +1205,16 @@ Remove an object version locked with retention mode `GOVERNANCE` using the `gove

<a name="removeObjects"></a>

### removeObjects(bucketName, objectsList[, callback])
### removeObjects(bucketName, objectsList)

Remove all objects in the objectsList.

**Parameters**

| Param | Type | Description |
| --------------- | ---------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `bucketName` | _string_ | Name of the bucket. |
| `objectsList` | _object_ | list of objects in the bucket to be removed. any one of the formats: 1. List of Object names as array of strings which are object keys: `['objectname1','objectname2']` 2. List of Object name and VersionId as an object: [{name:"my-obj-name",versionId:"my-versionId"}] |
| `callback(err)` | _function_ | Callback function is called with non `null` value in case of error. |
| Param | Type | Description |
| ------------- | -------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `bucketName` | _string_ | Name of the bucket. |
| `objectsList` | _object_ | list of objects in the bucket to be removed. any one of the formats: 1. List of Object names as array of strings which are object keys: `['objectname1','objectname2']` 2. List of Object name and VersionId as an object: [{name:"my-obj-name",versionId:"my-versionId"}] |

**Example**

Expand All @@ -1234,13 +1232,9 @@ objectsStream.on('error', function (e) {
console.log(e)
})

objectsStream.on('end', function () {
s3Client.removeObjects('my-bucketname', objectsList, function (e) {
if (e) {
return console.log('Unable to remove Objects ', e)
}
console.log('Removed the objects successfully')
})
objectsStream.on('end', async () => {
const remResult = await s3Client.removeObjects(bucket, objectsList)
console.log('Remove Errors', remResult.length)
})
```

Expand All @@ -1261,13 +1255,9 @@ objectsStream.on('data', function (obj) {
objectsStream.on('error', function (e) {
return console.log(e)
})
objectsStream.on('end', function () {
s3Client.removeObjects(bucket, objectsList, function (e) {
if (e) {
return console.log(e)
}
console.log('Success')
})
objectsStream.on('end', async () => {
const remResult = await s3Client.removeObjects(bucket, objectsList)
console.log('Remove Errors', remResult.length)
})
```

Expand Down
22 changes: 6 additions & 16 deletions examples/remove-objects.js
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ import * as Minio from 'minio'

const s3Client = new Minio.Client({
endPoint: 's3.amazonaws.com',
port: 9000,
useSSL: false,
accessKey: 'YOUR-ACCESSKEYID',
secretKey: 'YOUR-SECRETACCESSKEY',
})
Expand All @@ -48,32 +46,24 @@ function removeObjects(bucketName, prefix, recursive, includeVersion) {
return console.log(e)
})

objectsStream.on('end', function () {
s3Client.removeObjects(bucketName, objectsList, function (e) {
if (e) {
return console.log(e)
}
console.log('Success')
})
objectsStream.on('end', async () => {
const delRes = await s3Client.removeObjects(bucketName, objectsList)
console.log(delRes)
})
}

removeObjects(bucketName, prefix, recursive, true) // Versioned objects of a bucket to be deleted.
removeObjects(bucketName, prefix, recursive, false) // Normal objects of a bucket to be deleted.

// Delete Multiple objects and respective versions.
function removeObjectsMultipleVersions() {
async function removeObjectsMultipleVersions() {
const deleteList = [
{ versionId: '03ed08e1-34ff-4465-91ed-ba50c1e80f39', name: 'prefix-1/out.json.gz' },
{ versionId: '35517ae1-18cb-4a21-9551-867f53a10cfe', name: 'dir1/dir2/test.pdf' },
{ versionId: '3053f564-9aea-4a59-88f0-7f25d6320a2c', name: 'dir1/dir2/test.pdf' },
]

s3Client.removeObjects('my-bucket', deleteList, function (e) {
if (e) {
return console.log(e)
}
console.log('Successfully deleted..')
})
const delRes = await s3Client.removeObjects('my-bucket', deleteList)
console.log(delRes)
}
removeObjectsMultipleVersions()
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

58 changes: 41 additions & 17 deletions src/internal/client.ts
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,9 @@ import type {
ObjectMetaData,
PutObjectLegalHoldOptions,
PutTaggingParams,
RemoveObjectsParam,
RemoveObjectsRequestEntry,
RemoveObjectsResponse,
RemoveTaggingParams,
ReplicationConfig,
ReplicationConfigOpts,
Expand All @@ -88,13 +91,13 @@ import type {
VersionIdentificator,
} from './type.ts'
import type { ListMultipartResult, UploadedPart } from './xml-parser.ts'
import * as xmlParsers from './xml-parser.ts'
import {
parseCompleteMultipart,
parseInitiateMultipart,
parseObjectLegalHoldConfig,
parseSelectObjectContentResponse,
} from './xml-parser.ts'
import * as xmlParsers from './xml-parser.ts'

const xml = new xml2js.Builder({ renderOpts: { pretty: false }, headless: true })

Expand Down Expand Up @@ -1109,19 +1112,7 @@ export class TypedClient {
}
}

/**
* Remove the specified object.
* @deprecated use new promise style API
*/
removeObject(bucketName: string, objectName: string, removeOpts: RemoveOptions, callback: NoResultCallback): void
/**
* @deprecated use new promise style API
*/
// @ts-ignore
removeObject(bucketName: string, objectName: string, callback: NoResultCallback): void
async removeObject(bucketName: string, objectName: string, removeOpts?: RemoveOptions): Promise<void>

async removeObject(bucketName: string, objectName: string, removeOpts: RemoveOptions = {}): Promise<void> {
async removeObject(bucketName: string, objectName: string, removeOpts?: RemoveOptions): Promise<void> {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError(`Invalid bucket name: ${bucketName}`)
}
Expand All @@ -1136,15 +1127,15 @@ export class TypedClient {
const method = 'DELETE'

const headers: RequestHeaders = {}
if (removeOpts.governanceBypass) {
if (removeOpts?.governanceBypass) {
headers['X-Amz-Bypass-Governance-Retention'] = true
}
if (removeOpts.forceDelete) {
if (removeOpts?.forceDelete) {
headers['x-minio-force-delete'] = true
}

const queryParams: Record<string, string> = {}
if (removeOpts.versionId) {
if (removeOpts?.versionId) {
queryParams.versionId = `${removeOpts.versionId}`
}
const query = qs.stringify(queryParams)
Expand Down Expand Up @@ -2400,4 +2391,37 @@ export class TypedClient {

await this.makeRequestAsyncOmit({ method, bucketName, query }, '', [204])
}

async removeObjects(bucketName: string, objectsList: RemoveObjectsParam): Promise<RemoveObjectsResponse[]> {
if (!isValidBucketName(bucketName)) {
throw new errors.InvalidBucketNameError('Invalid bucket name: ' + bucketName)
}
if (!Array.isArray(objectsList)) {
throw new errors.InvalidArgumentError('objectsList should be a list')
}

const runDeleteObjects = async (batch: RemoveObjectsParam): Promise<RemoveObjectsResponse[]> => {
const delObjects: RemoveObjectsRequestEntry[] = batch.map((value) => {
return isObject(value) ? { Key: value.name, VersionId: value.versionId } : { Key: value }
})

const remObjects = { Delete: { Quiet: true, Object: delObjects } }
const payload = new xml2js.Builder({ headless: true }).buildObject(remObjects)
const headers: RequestHeaders = { 'Content-MD5': toMd5(payload) }

const res = await this.makeRequestAsync({ method: 'POST', bucketName, query: 'delete', headers }, payload)
const body = await readAsString(res)
return xmlParsers.removeObjectsParser(body)
}

const maxEntries = 1000 // max entries accepted in server for DeleteMultipleObjects API.
// Client side batching
const batches = []
for (let i = 0; i < objectsList.length; i += maxEntries) {
batches.push(objectsList.slice(i, i + maxEntries))
}

const batchResults = await Promise.all(batches.map(runDeleteObjects))
return batchResults.flat()
}
}
25 changes: 25 additions & 0 deletions src/internal/type.ts
Original file line number Diff line number Diff line change
Expand Up @@ -382,3 +382,28 @@ export type EncryptionRule = {
export type EncryptionConfig = {
Rule: EncryptionRule[]
}

export type RemoveObjectsEntry = {
name: string
versionId?: string
}
export type ObjectName = string

export type RemoveObjectsParam = ObjectName[] | RemoveObjectsEntry[]

export type RemoveObjectsRequestEntry = {
Key: string
VersionId?: string
}

export type RemoveObjectsResponse =
| null
| undefined
| {
Error?: {
Code?: string
Message?: string
Key?: string
VersionId?: string
}
}
9 changes: 9 additions & 0 deletions src/internal/xml-parser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -548,3 +548,12 @@ export function parseLifecycleConfig(xml: string) {
export function parseBucketEncryptionConfig(xml: string) {
return parseXml(xml)
}

export function removeObjectsParser(xml: string) {
const xmlObj = parseXml(xml)
if (xmlObj.DeleteResult && xmlObj.DeleteResult.Error) {
// return errors as array always. as the response is object in case of single object passed in removeObjects
return toArray(xmlObj.DeleteResult.Error)
}
return []
}
3 changes: 0 additions & 3 deletions src/minio.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -162,9 +162,6 @@ export class Client extends TypedClient {
conditions: CopyConditions,
): Promise<BucketItemCopy>

removeObjects(bucketName: string, objectsList: string[], callback: NoResultCallback): void
removeObjects(bucketName: string, objectsList: string[]): Promise<void>

removeIncompleteUpload(bucketName: string, objectName: string, callback: NoResultCallback): void
removeIncompleteUpload(bucketName: string, objectName: string): Promise<void>

Expand Down
Loading

0 comments on commit 5ef8ae1

Please sign in to comment.