From 875bbb862f5f34287b6cd8884943d3e0fa129b80 Mon Sep 17 00:00:00 2001 From: David An Date: Thu, 16 May 2024 09:28:06 -0400 Subject: [PATCH] AJ-1730: sunset BDBag support (#1353) --- project/Dependencies.scala | 2 - src/main/resources/reference.conf | 1 - src/main/resources/swagger/api-docs.yaml | 48 ------- .../dsde/firecloud/EntityService.scala | 130 ------------------ .../dsde/firecloud/FireCloudConfig.scala | 1 - .../firecloud/model/ModelJsonProtocol.scala | 1 - .../dsde/firecloud/model/Workspace.scala | 2 - .../webservice/WorkspaceApiService.scala | 12 +- .../dsde/firecloud/EntityServiceSpec.scala | 79 ----------- .../dsde/firecloud/mock/MockUtils.scala | 1 - .../webservice/WorkspaceApiServiceSpec.scala | 93 ------------- 11 files changed, 1 insertion(+), 369 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 1d3fe2322..489e6d4af 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -93,8 +93,6 @@ object Dependencies { "org.scalatest" %% "scalatest" % "3.2.18" % "test", "org.mock-server" % "mockserver-netty" % "5.15.0" % "test", - // jaxb-api needed by WorkspaceApiServiceSpec.bagitService() method - "javax.xml.bind" % "jaxb-api" % "2.3.1" % "test", // provides testing mocks "com.google.cloud" % "google-cloud-nio" % "0.127.17" % "test", "org.scalatestplus" %% "mockito-4-5" % "3.2.12.0" % "test" diff --git a/src/main/resources/reference.conf b/src/main/resources/reference.conf index e2f10f47a..14822445d 100644 --- a/src/main/resources/reference.conf +++ b/src/main/resources/reference.conf @@ -9,7 +9,6 @@ workspace { workspacesAclPath="/workspaces/%s/%s/acl" entitiesPath="/workspaces/%s/%s/entities" entityQueryPath="/workspaces/%s/%s/entityQuery" - entityBagitMaximumSize=200000000 importEntitiesPath="/workspaces/%s/%s/importEntities" workspacesEntitiesCopyPath="/workspaces/entities/copy" submissionsPath="/workspaces/%s/%s/submissions" diff --git a/src/main/resources/swagger/api-docs.yaml b/src/main/resources/swagger/api-docs.yaml index 86fd556fc..7e2f07198 100755 --- a/src/main/resources/swagger/api-docs.yaml +++ b/src/main/resources/swagger/api-docs.yaml @@ -3775,39 +3775,6 @@ paths: description: Internal Error content: {} x-passthrough: false - /api/workspaces/{workspaceNamespace}/{workspaceName}/importBagit: - post: - tags: - - Entities - deprecated: true - summary: Import entity TSVs from a zipped [BagIt](https://tools.ietf.org/html/draft-kunze-bagit-14) - directory, whose payload contains two files - participants.tsv and samples.tsv - operationId: importBagit - parameters: - - $ref: '#/components/parameters/workspaceNamespaceParam' - - $ref: '#/components/parameters/workspaceNameParam' - requestBody: - description: JSON object containing bagit URL - content: - 'application/json': - schema: - $ref: '#/components/schemas/BagitRequest' - required: true - responses: - 200: - description: Successful Request - content: {} - 400: - description: Bad Request - content: {} - 404: - description: Source Workspace not found - content: {} - 500: - description: Internal Error - content: {} - x-passthrough: false - x-codegen-request-body-name: bagitImportRequest /api/workspaces/{workspaceNamespace}/{workspaceName}/importEntities: post: tags: @@ -6415,21 +6382,6 @@ components: service format: int32 description: Stuff you need to know about calls - BagitRequest: - required: - - bagitURL - - format - type: object - properties: - bagitURL: - type: string - description: link to publically accessible zipped BagIt directory - format: - type: string - description: the type of the files inside the bagit. Currently this must - be the string "TSV". - enum: - - TSV ConfigurationIngest: required: - inputs diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/EntityService.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/EntityService.scala index 283b1881f..ac1123e4f 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/EntityService.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/EntityService.scala @@ -4,7 +4,6 @@ import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.model.{HttpResponse, StatusCodes} import com.typesafe.scalalogging.LazyLogging import org.broadinstitute.dsde.firecloud.EntityService._ -import org.broadinstitute.dsde.firecloud.FireCloudConfig.Rawls import org.broadinstitute.dsde.firecloud.dataaccess.ImportServiceFiletypes.FILETYPE_RAWLS import org.broadinstitute.dsde.firecloud.dataaccess.{CwdsDAO, GoogleServicesDAO, ImportServiceDAO, RawlsDAO} import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ @@ -18,16 +17,9 @@ import org.broadinstitute.dsde.workbench.model.google.{GcsBucketName, GcsObjectN import org.databiosphere.workspacedata.client.ApiException import spray.json.DefaultJsonProtocol._ -import java.io.{File, FileNotFoundException, FileOutputStream, InputStream} -import java.net.{HttpURLConnection, URL} -import java.nio.channels.Channels import java.nio.charset.StandardCharsets import java.text.SimpleDateFormat -import java.util.concurrent.atomic.AtomicLong -import java.util.zip.{ZipEntry, ZipException, ZipFile} import scala.concurrent.{ExecutionContext, Future} -import scala.io.Source -import scala.jdk.CollectionConverters._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} @@ -56,50 +48,6 @@ object EntityService { } } - //returns (contents of participants.tsv, contents of samples.tsv) - def unzipTSVs(bagName: String, zipFile: ZipFile)(op: (Option[String], Option[String]) => Future[PerRequestMessage]): Future[PerRequestMessage] = { - val zipEntries = zipFile.entries.asScala - - val rand = java.util.UUID.randomUUID.toString.take(8) - val participantsTmp = File.createTempFile(s"$rand-participants", ".tsv") - val samplesTmp = File.createTempFile(s"$rand-samples", ".tsv") - - val unzippedFiles = zipEntries.foldLeft((None: Option[String], None: Option[String])){ (acc: (Option[String], Option[String]), ent: ZipEntry) => - if(!ent.isDirectory && (ent.getName.contains("/participants.tsv") || ent.getName.equals("participants.tsv"))) { - acc._1 match { - case Some(_) => throw new FireCloudExceptionWithErrorReport(errorReport = ErrorReport(StatusCodes.BadRequest, s"More than one participants.tsv file found in bagit $bagName")) - case None => - unzipSingleFile(zipFile.getInputStream(ent), participantsTmp) - (Some(participantsTmp.getPath), acc._2) - } - } else if(!ent.isDirectory && (ent.getName.contains("/samples.tsv") || ent.getName.equals("samples.tsv"))) { - acc._2 match { - case Some(_) => throw new FireCloudExceptionWithErrorReport(errorReport = ErrorReport(StatusCodes.BadRequest, s"More than one samples.tsv file found in bagit $bagName")) - case None => - unzipSingleFile (zipFile.getInputStream (ent), samplesTmp) - (acc._1, Some (samplesTmp.getPath) ) - } - } else { - acc - } - } - - try { - op(unzippedFiles._1.map(f => Source.fromFile(f).mkString), unzippedFiles._2.map(f => Source.fromFile(f).mkString)) - } catch { - case e: Exception => throw e - } finally { - participantsTmp.delete - samplesTmp.delete - } - } - - private def unzipSingleFile(zis: InputStream, fileTarget: File): Unit = { - val fout = new FileOutputStream(fileTarget) - val buffer = new Array[Byte](1024) - LazyList.continually(zis.read(buffer)).takeWhile(_ != -1).foreach(fout.write(buffer, 0, _)) - } - } class EntityService(rawlsDAO: RawlsDAO, importServiceDAO: ImportServiceDAO, cwdsDAO: CwdsDAO, googleServicesDAO: GoogleServicesDAO, modelSchema: ModelSchema)(implicit val executionContext: ExecutionContext) @@ -320,84 +268,6 @@ class EntityService(rawlsDAO: RawlsDAO, importServiceDAO: ImportServiceDAO, cwds } } - def importBagit(workspaceNamespace: String, workspaceName: String, bagitRq: BagitImportRequest, userInfo: UserInfo): Future[PerRequestMessage] = { - if(bagitRq.format != "TSV") { - Future.successful(RequestCompleteWithErrorReport(StatusCodes.BadRequest, "Invalid format; for now, you must place the string \"TSV\" here")) - } else { - - //Java URL handles http, https, ftp, file, and jar protocols. - //We're only OK with https to avoid MITM attacks. - val bagitURL = new URL(bagitRq.bagitURL.replace(" ", "%20")) - val acceptableProtocols = Seq("https") //for when we inevitably change our mind and need to support others - if (!acceptableProtocols.contains(bagitURL.getProtocol)) { - Future.successful(RequestCompleteWithErrorReport(StatusCodes.BadRequest, "Invalid bagitURL protocol: must be https only")) - } else { - - val rand = java.util.UUID.randomUUID.toString.take(8) - val bagItFile = File.createTempFile(s"$rand-samples", ".tsv") - var bytesDownloaded = new AtomicLong(-1) // careful, this is a var - - try { - val conn = bagitURL.openConnection() - val length = conn.getContentLength - conn.asInstanceOf[HttpURLConnection].disconnect() - - if (length == 0) { - Future.successful(RequestCompleteWithErrorReport(StatusCodes.BadRequest, s"BDBag has content-length 0")) - } else if (length > Rawls.entityBagitMaximumSize) { - Future.successful(RequestCompleteWithErrorReport(StatusCodes.BadRequest, s"BDBag size is too large.")) - } else { - // download the file - val readFromBagit = Channels.newChannel(bagitURL.openStream()) - val writeToTemp = new FileOutputStream(bagItFile) - try { - bytesDownloaded.set(writeToTemp.getChannel.transferFrom(readFromBagit, 0, length)) - } finally { - readFromBagit.close() - writeToTemp.close() - } - - val zipFile = new ZipFile(bagItFile.getAbsolutePath) - if (!zipFile.entries().hasMoreElements) { - Future(RequestCompleteWithErrorReport(StatusCodes.BadRequest, s"BDBag has no entries.")) - } else { - //make two big strings containing the participants and samples TSVs - //if i could turn back time this would use streams to save memory, but hopefully this will all go away when entity service comes along - unzipTSVs(bagitRq.bagitURL, zipFile) { (participantsStr, samplesStr) => - (participantsStr, samplesStr) match { - case (None, None) => - Future.successful(RequestCompleteWithErrorReport(StatusCodes.BadRequest, "You must have either (or both) participants.tsv and samples.tsv in the zip file")) - case _ => - for { - // This should vomit back errors from rawls. - participantResult <- participantsStr.map(ps => importEntitiesFromTSV(workspaceNamespace, workspaceName, ps, userInfo)).getOrElse(Future.successful(RequestComplete(OK))) - sampleResult <- samplesStr.map(ss => importEntitiesFromTSV(workspaceNamespace, workspaceName, ss, userInfo)).getOrElse(Future.successful(RequestComplete(OK))) - } yield { - participantResult match { - case RequestComplete((OK, _)) => sampleResult - case _ => participantResult - } - } - } - } - } - } - } catch { - case _:FileNotFoundException => - Future.successful(RequestCompleteWithErrorReport(StatusCodes.NotFound, s"BDBag ${bagitRq.bagitURL} was not found.")) - case ze:ZipException => - logger.info(s"ZipException: ${ze.getMessage} - ${bagItFile.getAbsolutePath} has length ${bagItFile.length}. " + - s"We originally downloaded $bytesDownloaded bytes.") - Future.successful(RequestCompleteWithErrorReport(StatusCodes.BadRequest, s"Problem with BDBag: ${ze.getMessage}")) - case e: Exception => - throw e - } finally { - bagItFile.delete() - } - } - } - } - def importJob(workspaceNamespace: String, workspaceName: String, importRequest: AsyncImportRequest, userInfo: UserInfo): Future[PerRequestMessage] = { // validate that filetype exists in the importRequest if (importRequest.filetype.isEmpty) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala index bfdf5a4d7..3532e9fe5 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala @@ -49,7 +49,6 @@ object FireCloudConfig { val workspacesUrl = authUrl + workspacesPath val entitiesPath = workspace.getString("entitiesPath") val entityQueryPath = workspace.getString("entityQueryPath") - val entityBagitMaximumSize = workspace.getInt("entityBagitMaximumSize") val workspacesEntitiesCopyPath = workspace.getString("workspacesEntitiesCopyPath") def workspacesEntitiesCopyUrl(linkExistingEntities: Boolean) = authUrl + workspacesEntitiesCopyPath + "?linkExistingEntities=%s".format(linkExistingEntities) val submissionsCountPath = workspace.getString("submissionsCountPath") diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala index 4418ac8b6..f6f47a01d 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala @@ -237,7 +237,6 @@ object ModelJsonProtocol extends WorkspaceJsonSupport with SprayJsonSupport { implicit val impCurator: RootJsonFormat[Curator] = jsonFormat1(Curator) implicit val impUserImportPermission: RootJsonFormat[UserImportPermission] = jsonFormat2(UserImportPermission) - implicit val impBagitImportRequest: RootJsonFormat[BagitImportRequest] = jsonFormat2(BagitImportRequest) implicit val impPFBImportRequest: RootJsonFormat[PFBImportRequest] = jsonFormat1(PFBImportRequest) implicit val impOptions: RootJsonFormat[ImportOptions] = jsonFormat2(ImportOptions) implicit val impAsyncImportRequest: RootJsonFormat[AsyncImportRequest] = jsonFormat3(AsyncImportRequest) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/model/Workspace.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/model/Workspace.scala index c2bad17a7..41ada7031 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/model/Workspace.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/model/Workspace.scala @@ -22,8 +22,6 @@ case class EntityCopyWithoutDestinationDefinition( case class EntityId(entityType: String, entityName: String) -case class BagitImportRequest(bagitURL: String, format: String) - // legacy class specific to PFB import; prefer AsyncImportRequest instead case class PFBImportRequest(url: String) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiService.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiService.scala index 40a983e60..d704a12bf 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiService.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiService.scala @@ -1,6 +1,6 @@ package org.broadinstitute.dsde.firecloud.webservice -import akka.http.scaladsl.model.StatusCodes.{Accepted, OK} +import akka.http.scaladsl.model.StatusCodes.OK import java.text.SimpleDateFormat import akka.http.scaladsl.model.Uri.Query @@ -22,7 +22,6 @@ import org.slf4j.{Logger, LoggerFactory} import spray.json.DefaultJsonProtocol._ import scala.concurrent.ExecutionContext -import scala.util.Try trait WorkspaceApiService extends FireCloudRequestBuilding with FireCloudDirectives with StandardUserInfoDirectives { @@ -146,15 +145,6 @@ trait WorkspaceApiService extends FireCloudRequestBuilding with FireCloudDirecti } } } ~ - path("importBagit"){ - post { - requireUserInfo() { userInfo => - entity(as[BagitImportRequest]) { bagitRq => - complete { entityServiceConstructor(FirecloudModelSchema).importBagit(workspaceNamespace, workspaceName, bagitRq, userInfo) } - } - } - } - } ~ // POST importPFB will likely be deprecated in the future; use POST importJob instead path("importPFB") { post { diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/EntityServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/EntityServiceSpec.scala index 5ac83a1d0..679cb8e2b 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/EntityServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/EntityServiceSpec.scala @@ -23,7 +23,6 @@ import org.scalatest.concurrent.ScalaFutures import org.scalatestplus.mockito.MockitoSugar.{mock => mockito} import java.util.UUID -import java.util.zip.ZipFile import scala.concurrent.ExecutionContext.Implicits._ import scala.concurrent.Future import scala.reflect.classTag @@ -42,84 +41,6 @@ class EntityServiceSpec extends BaseServiceSpec with BeforeAndAfterEach { private def dummyUserInfo(tokenStr: String) = UserInfo("dummy", OAuth2BearerToken(tokenStr), -1, "dummy") - "EntityClient should extract TSV files out of bagit zips" - { - "with neither participants nor samples in the zip" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/nothingbag.zip") - EntityService.unzipTSVs("nothingbag", zip) { (participants, samples) => - participants shouldBe None - samples shouldBe None - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - - "with both participants and samples in a zip with a flat file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/flat_testbag.zip") - EntityService.unzipTSVs("flat_testbag", zip) { (participants, samples) => - participants.map(_.stripLineEnd) shouldEqual Some("imagine this is a participants.tsv in a flat file structure") - samples.map(_.stripLineEnd) shouldEqual Some("imagine this is a samples.tsv in a flat file structure") - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - - "with only participants in a zip with a flat file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/participants_only_flat_testbag.zip") - EntityService.unzipTSVs("participants_only_flat_testbag", zip) { (participants, samples) => - participants.map(_.stripLineEnd) shouldEqual Some("imagine this is a participants.tsv all alone in a flat file structure") - samples.map(_.stripLineEnd) shouldEqual None - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - - "with only samples in a zip with a flat file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/samples_only_flat_testbag.zip") - EntityService.unzipTSVs("samples_only_flat_testbag", zip) { (participants, samples) => - participants.map(_.stripLineEnd) shouldEqual None - samples.map(_.stripLineEnd) shouldEqual Some("imagine this is a samples.tsv all alone in a flat file structure") - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - - "with both participants and samples in a zip with a nested file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/nested_testbag.zip") - EntityService.unzipTSVs("nested_testbag", zip) { (participants, samples) => - participants.map(_.stripLineEnd) shouldEqual Some("imagine this is a participants.tsv in a nested file structure") - samples.map(_.stripLineEnd) shouldEqual Some("imagine this is a samples.tsv in a nested file structure") - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - - "with both participants and samples and an extra tsv file in a zip with a nested file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/extra_file_nested_testbag.zip") - EntityService.unzipTSVs("extra_file_nested_testbag", zip) { (participants, samples) => - participants.map(_.stripLineEnd) shouldEqual Some("imagine this is a participants.tsv in a nested file structure") - samples.map(_.stripLineEnd) shouldEqual Some("imagine this is a samples.tsv in a nested file structure") - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - - "with multiple participants in a zip with a flat file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/duplicate_participants_nested_testbag.zip") - - val ex = intercept[FireCloudExceptionWithErrorReport] { - EntityService.unzipTSVs("duplicate_participants_nested_testbag", zip) { (_, _) => - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - ex.errorReport.message shouldEqual "More than one participants.tsv file found in bagit duplicate_participants_nested_testbag" - } - - "with multiple samples in a zip with a flat file structure" in { - val zip = new ZipFile("src/test/resources/testfiles/bagit/duplicate_samples_nested_testbag.zip") - - val ex = intercept[FireCloudExceptionWithErrorReport] { - EntityService.unzipTSVs("duplicate_samples_nested_testbag", zip) { (_, _) => - Future.successful(RequestComplete(StatusCodes.OK)) - } - } - ex.errorReport.message shouldEqual "More than one samples.tsv file found in bagit duplicate_samples_nested_testbag" - } - } - "EntityService.importEntitiesFromTSV()" - { val tsvParticipants = FileUtils.readAllTextFromResource("testfiles/tsv/ADD_PARTICIPANTS.txt") val tsvMembership = FileUtils.readAllTextFromResource("testfiles/tsv/MEMBERSHIP_SAMPLE_SET.tsv") diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/mock/MockUtils.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/mock/MockUtils.scala index 126cf8d00..6aa39fe2b 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/mock/MockUtils.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/mock/MockUtils.scala @@ -19,7 +19,6 @@ object MockUtils { val ontologyServerPort = 8993 val samServerPort = 8994 val cromiamServerPort = 8995 - val bagitServerPort = 9393 val importServiceServerPort = 9394 def randomPositiveInt(): Int = { diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiServiceSpec.scala index 1b9c6ec46..ba3b6a20e 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/WorkspaceApiServiceSpec.scala @@ -1,14 +1,11 @@ package org.broadinstitute.dsde.firecloud.webservice -import java.util.UUID import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.model.Uri.Query import akka.http.scaladsl.model._ import akka.http.scaladsl.server.Route.{seal => sealRoute} -import javax.net.ssl.HttpsURLConnection -import org.apache.commons.io.IOUtils import org.broadinstitute.dsde.firecloud.dataaccess.ImportServiceFiletypes.{FILETYPE_PFB, FILETYPE_TDR} import org.broadinstitute.dsde.firecloud.dataaccess.{MockCwdsDAO, MockRawlsDAO, MockShareLogDAO, WorkspaceApiServiceSpecShareLogDAO} import org.broadinstitute.dsde.firecloud.mock.MockUtils._ @@ -20,13 +17,9 @@ import org.broadinstitute.dsde.firecloud.{EntityService, FireCloudConfig} import org.broadinstitute.dsde.rawls.model.WorkspaceACLJsonSupport._ import org.broadinstitute.dsde.rawls.model._ import org.joda.time.DateTime -import org.mockserver.configuration.Configuration import org.mockserver.integration.ClientAndServer import org.mockserver.integration.ClientAndServer._ -import org.mockserver.logging.MockServerLogger import org.mockserver.model.HttpRequest._ -import org.mockserver.model.Parameter -import org.mockserver.socket.tls.KeyStoreFactory import org.scalatest.BeforeAndAfterEach import spray.json.DefaultJsonProtocol._ import spray.json._ @@ -109,7 +102,6 @@ class WorkspaceApiServiceSpec extends BaseServiceSpec with WorkspaceApiService w private final val bucketPath = workspacesRoot + "/%s/%s/checkBucketReadAccess".format(workspace.namespace, workspace.name) private final val tsvImportPath = workspacesRoot + "/%s/%s/importEntities".format(workspace.namespace, workspace.name) private final val tsvImportFlexiblePath = workspacesRoot + "/%s/%s/flexibleImportEntities".format(workspace.namespace, workspace.name) - private final val bagitImportPath = workspacesRoot + "/%s/%s/importBagit".format(workspace.namespace, workspace.name) private final val pfbImportPath = workspacesRoot + "/%s/%s/importPFB".format(workspace.namespace, workspace.name) private final val importJobPath = workspacesRoot + "/%s/%s/importJob".format(workspace.namespace, workspace.name) private final val importJobStatusPath = workspacesRoot + "/%s/%s/importJob".format(workspace.namespace, workspace.name) @@ -212,7 +204,6 @@ class WorkspaceApiServiceSpec extends BaseServiceSpec with WorkspaceApiService w val nonAuthDomainRawlsWorkspaceResponse = WorkspaceResponse(Some(WorkspaceAccessLevels.Owner), canShare=Some(false), canCompute=Some(true), catalog=Some(false), nonAuthDomainRawlsWorkspace, Some(WorkspaceSubmissionStats(None, None, runningSubmissionsCount = 0)), Some(WorkspaceBucketOptions(false)), Some(Set.empty), None) var rawlsServer: ClientAndServer = _ - var bagitServer: ClientAndServer = _ var importServiceServer: ClientAndServer = _ /** Stubs the mock Rawls service to respond to a request. Used for testing passthroughs. @@ -289,46 +280,13 @@ class WorkspaceApiServiceSpec extends BaseServiceSpec with WorkspaceApiService w ) } - def bagitService() = { - val bothBytes = IOUtils.toByteArray(getClass.getClassLoader.getResourceAsStream("testfiles/bagit/testbag.zip")) - val neitherBytes = IOUtils.toByteArray(getClass.getClassLoader.getResourceAsStream("testfiles/bagit/nothingbag.zip")) - val emptyBytes = IOUtils.toByteArray(getClass.getClassLoader.getResourceAsStream("testfiles/bagit/empty.zip")) - val notAZipBytes = IOUtils.toByteArray(getClass.getClassLoader.getResourceAsStream("testfiles/bagit/not_a_zip.txt")) - - // url -> byte array for mockserver - val mappings = Map( - "/both.zip" -> bothBytes, - "/neither.zip" -> neitherBytes, - "/empty.zip" -> emptyBytes, - "/notazip.zip" -> notAZipBytes, - ) - - // bagit import requires https urls; set up SSL - HttpsURLConnection.setDefaultSSLSocketFactory(new KeyStoreFactory( - Configuration.configuration(), - new MockServerLogger()).sslContext().getSocketFactory) - - // set up mockserver for all paths defined above - mappings.foreach { entry => - bagitServer - .when(request().withMethod("GET").withPath(entry._1)) - .respond( - org.mockserver.model.HttpResponse.response() - .withStatusCode(200) - .withBody(org.mockserver.model.BinaryBody.binary(entry._2))) - } - } - override def beforeAll(): Unit = { rawlsServer = startClientAndServer(MockUtils.workspaceServerPort) - bagitServer = startClientAndServer(MockUtils.bagitServerPort) - bagitService() importServiceServer = startClientAndServer(MockUtils.importServiceServerPort) } override def afterAll(): Unit = { rawlsServer.stop - bagitServer.stop importServiceServer.stop } @@ -1056,57 +1014,6 @@ class WorkspaceApiServiceSpec extends BaseServiceSpec with WorkspaceApiService w } } - "WorkspaceService BagIt Tests" - { - "should unbundle a bagit containing both participants and samples" in { - stubRawlsService(HttpMethods.POST, s"$workspacesPath/entities/batchUpsert", NoContent) - (Post(bagitImportPath, HttpEntity(MediaTypes.`application/json`, s"""{"bagitURL":"https://localhost:$bagitServerPort/both.zip", "format":"TSV" }""")) - ~> dummyUserIdHeaders(dummyUserId) - ~> sealRoute(workspaceRoutes)) ~> check { - status should equal(OK) - } - } - - "should 400 if a bagit doesn't have either participants or samples" in { - stubRawlsService(HttpMethods.POST, s"$workspacesPath/entities/batchUpsert", NoContent) - (Post(bagitImportPath, HttpEntity(MediaTypes.`application/json`, s"""{"bagitURL":"https://localhost:$bagitServerPort/neither.zip", "format":"TSV" }""")) - ~> dummyUserIdHeaders(dummyUserId) - ~> sealRoute(workspaceRoutes)) ~> check { - status should equal(BadRequest) - } - } - - "should 400 if a bagit request has an invalid format" in { - stubRawlsService(HttpMethods.POST, s"$workspacesPath/entities/batchUpsert", NoContent) - (Post(bagitImportPath, HttpEntity(MediaTypes.`application/json`, s"""{"bagitURL":"https://localhost:$bagitServerPort/both.zip", "format":"garbage" }""")) - ~> dummyUserIdHeaders(dummyUserId) - ~> sealRoute(workspaceRoutes)) ~> check { - status should equal(BadRequest) - } - } - - "should 400 if a bagit is empty" in { - stubRawlsService(HttpMethods.POST, s"$workspacesPath/entities/batchUpsert", NoContent) - (Post(bagitImportPath, HttpEntity(MediaTypes.`application/json`, s"""{"bagitURL":"https://localhost:$bagitServerPort/empty.zip", "format":"TSV" }""")) - ~> dummyUserIdHeaders(dummyUserId) - ~> sealRoute(workspaceRoutes)) ~> check { - status should equal(BadRequest) - val errRpt = responseAs[ErrorReport] - errRpt.message shouldBe s"BDBag has no entries." - } - } - - "should 400 if a bagit is unzippable" in { - stubRawlsService(HttpMethods.POST, s"$workspacesPath/entities/batchUpsert", NoContent) - (Post(bagitImportPath, HttpEntity(MediaTypes.`application/json`, s"""{"bagitURL":"https://localhost:$bagitServerPort/notazip.zip", "format":"TSV" }""")) - ~> dummyUserIdHeaders(dummyUserId) - ~> sealRoute(workspaceRoutes)) ~> check { - status should equal(BadRequest) - val errRpt = responseAs[ErrorReport] - errRpt.message shouldBe s"Problem with BDBag: zip END header not found" - } - } - } - "WorkspaceService importPFB Tests" - { "should 400 if import service indicates a bad request" in {