diff --git a/local-dev/templates/docker-rsync-local-orch.sh b/local-dev/templates/docker-rsync-local-orch.sh index b71ad92b7..bd7d0dc08 100755 --- a/local-dev/templates/docker-rsync-local-orch.sh +++ b/local-dev/templates/docker-rsync-local-orch.sh @@ -112,7 +112,7 @@ start_server () { --network=fc-orch \ -e JAVA_OPTS="$DOCKER_JAVA_OPTS" \ sbtscala/scala-sbt:eclipse-temurin-jammy-17.0.10_7_1.10.1_2.13.14 \ - sbt \~reStart + bash -c "git config --global --add safe.directory /app && sbt \~reStart" docker cp config/firecloud-account.pem orch-sbt:/etc/firecloud-account.pem docker cp config/firecloud-account.json orch-sbt:/etc/firecloud-account.json diff --git a/local-dev/templates/firecloud-orchestration.conf.ctmpl b/local-dev/templates/firecloud-orchestration.conf.ctmpl index d19d55dcc..2c33ce91d 100644 --- a/local-dev/templates/firecloud-orchestration.conf.ctmpl +++ b/local-dev/templates/firecloud-orchestration.conf.ctmpl @@ -105,6 +105,11 @@ cwds { bucketName = "cwds-batchupsert-dev" } +externalCreds { + baseUrl = "https://externalcreds.dsde-dev.broadinstitute.org" + enabled = true +} + firecloud { baseUrl = "https://firecloud-orchestration.dsde-dev.broadinstitute.org" portalUrl = "https://firecloud.dsde-dev.broadinstitute.org" diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 5a68d2ccf..7bb7c46fd 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -14,6 +14,10 @@ object Dependencies { val excludeAkkaHttp = ExclusionRule(organization = "com.typesafe.akka", name = "akka-http_2.13") val excludeSprayJson = ExclusionRule(organization = "com.typesafe.akka", name = "akka-http-spray-json_2.13") + val excludeSpring = ExclusionRule(organization = "org.springframework") + val excludeSpringBoot = ExclusionRule(organization = "org.springframework.boot") + val excludeSpringJcl = ExclusionRule(organization = "org.springframework", name = "spring-jcl") + // Overrides for transitive dependencies. These apply - via Settings.scala - to all projects in this codebase. // These are overrides only; if the direct dependencies stop including any of these, they will not be included // by being listed here. @@ -50,9 +54,11 @@ object Dependencies { excludeGuava("org.broadinstitute.dsde.workbench" %% "workbench-util" % s"0.10-$workbenchLibsHash"), "org.broadinstitute.dsde.workbench" %% "workbench-google2" % s"0.36-$workbenchLibsHash", "org.broadinstitute.dsde.workbench" %% "workbench-oauth2" % s"0.7-$workbenchLibsHash", - "org.broadinstitute.dsde.workbench" %% "sam-client" % "0.1-ef83073", + "org.broadinstitute.dsde.workbench" %% "sam-client" % "v0.0.263", "org.broadinstitute.dsde.workbench" %% "workbench-notifications" %s"0.6-$workbenchLibsHash", "org.databiosphere" % "workspacedataservice-client-okhttp-jakarta" % "0.2.167-SNAPSHOT", + "bio.terra" % "externalcreds-client-resttemplate" % "1.44.0-20240725.201427-1" excludeAll(excludeSpring, excludeSpringBoot), + "org.springframework" % "spring-web" % "6.1.11" excludeAll(excludeSpringBoot, excludeSpringJcl), "com.typesafe.akka" %% "akka-actor" % akkaV, "com.typesafe.akka" %% "akka-slf4j" % akkaV, diff --git a/project/Merging.scala b/project/Merging.scala index 56b5d5172..08bcc72f2 100644 --- a/project/Merging.scala +++ b/project/Merging.scala @@ -24,6 +24,7 @@ object Merging { // Error: /home/sbtuser/.cache/coursier/v1/https/repo1.maven.org/maven2/com/typesafe/akka/akka-protobuf-v3_2.13/2.6.19/akka-protobuf-v3_2.13-2.6.19.jar:google/protobuf/struct.proto case PathList("google", "protobuf", _ @ _*) => MergeStrategy.first case PathList("META-INF", "versions", "9", "OSGI-INF", "MANIFEST.MF") => MergeStrategy.first + case PathList("META-INF", "spring", "aot.factories") => MergeStrategy.first case x => oldStrategy(x) } } diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/Application.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/Application.scala index dd1800530..692832be8 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/Application.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/Application.scala @@ -16,4 +16,5 @@ case class Application(agoraDAO: AgoraDAO, thurloeDAO: ThurloeDAO, shareLogDAO: ShareLogDAO, shibbolethDAO: ShibbolethDAO, - cwdsDAO: CwdsDAO) + cwdsDAO: CwdsDAO, + ecmDAO: ExternalCredsDAO) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/Boot.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/Boot.scala index 17f63c4b2..7cc09796f 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/Boot.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/Boot.scala @@ -94,6 +94,7 @@ object Boot extends App with LazyLogging { val rawlsDAO: RawlsDAO = new HttpRawlsDAO val samDAO: SamDAO = new HttpSamDAO val thurloeDAO: ThurloeDAO = new HttpThurloeDAO + val ecmDAO: ExternalCredsDAO = if (FireCloudConfig.ExternalCreds.enabled) new HttpExternalCredsDAO else new DisabledExternalCredsDAO // can be disabled val agoraDAO: AgoraDAO = whenEnabled[AgoraDAO](FireCloudConfig.Agora.enabled, new HttpAgoraDAO(FireCloudConfig.Agora)) @@ -110,7 +111,7 @@ object Boot extends App with LazyLogging { val searchDAO: SearchDAO = elasticSearchClient.map(new ElasticSearchDAO(_, FireCloudConfig.ElasticSearch.indexName, researchPurposeSupport)).getOrElse(DisabledServiceFactory.newDisabledService[SearchDAO]) val shareLogDAO: ShareLogDAO = elasticSearchClient.map(new ElasticSearchShareLogDAO(_, FireCloudConfig.ElasticSearch.shareLogIndexName)).getOrElse(DisabledServiceFactory.newDisabledService[ShareLogDAO]) - Application(agoraDAO, googleServicesDAO, ontologyDAO, rawlsDAO, samDAO, searchDAO, researchPurposeSupport, thurloeDAO, shareLogDAO, shibbolethDAO, cwdsDAO) + Application(agoraDAO, googleServicesDAO, ontologyDAO, rawlsDAO, samDAO, searchDAO, researchPurposeSupport, thurloeDAO, shareLogDAO, shibbolethDAO, cwdsDAO, ecmDAO) } private def whenEnabled[T : ClassTag](enabled: Boolean, realService: => T): T = { diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala index 368596ff5..71e01b1ce 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/FireCloudConfig.scala @@ -3,7 +3,7 @@ package org.broadinstitute.dsde.firecloud import akka.http.scaladsl.model.Uri import akka.http.scaladsl.model.Uri.{Authority, Host, Query} import com.typesafe.config.{Config, ConfigFactory, ConfigObject} -import org.broadinstitute.dsde.firecloud.service.{FireCloudDirectiveUtils, NihWhitelist} +import org.broadinstitute.dsde.firecloud.service.{FireCloudDirectiveUtils, NihAllowlist} import org.broadinstitute.dsde.rawls.model.{EntityQuery, SortDirections} import org.broadinstitute.dsde.workbench.model.WorkbenchGroupName @@ -125,6 +125,13 @@ object FireCloudConfig { lazy val bucket: String = cwds.getString("bucketName") } + object ExternalCreds { + // lazy - only required when External Credentials is enabled + private lazy val externalCreds = config.getConfig("externalCreds") + lazy val baseUrl: String = externalCreds.getString("baseUrl") + lazy val enabled: Boolean = externalCreds.getBoolean("enabled") + } + object FireCloud { private val firecloud = config.getConfig("firecloud") val fireCloudId = firecloud.getString("fireCloudId") @@ -147,7 +154,7 @@ object FireCloudConfig { // lazy - only required when nih is enabled private lazy val nih = config.getConfig("nih") lazy val whitelistBucket = nih.getString("whitelistBucket") - lazy val whitelists: Set[NihWhitelist] = { + lazy val whitelists: Set[NihAllowlist] = { val whitelistConfigs = nih.getConfig("whitelists") whitelistConfigs.root.asScala.collect { case (name, configObject:ConfigObject) => @@ -155,7 +162,7 @@ object FireCloudConfig { val rawlsGroup = config.getString("rawlsGroup") val fileName = config.getString("fileName") - NihWhitelist(name, WorkbenchGroupName(rawlsGroup), fileName) + NihAllowlist(name, WorkbenchGroupName(rawlsGroup), fileName) } }.toSet val enabled = nih.optionalBoolean("enabled").getOrElse(true) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/DisabledExternalCredsDAO.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/DisabledExternalCredsDAO.scala new file mode 100644 index 000000000..a5c032ed4 --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/DisabledExternalCredsDAO.scala @@ -0,0 +1,32 @@ +package org.broadinstitute.dsde.firecloud.dataaccess + +import com.typesafe.scalalogging.LazyLogging +import org.broadinstitute.dsde.firecloud.model.{LinkedEraAccount, UserInfo, WithAccessToken} + +import scala.concurrent.Future + +class DisabledExternalCredsDAO extends ExternalCredsDAO with LazyLogging { + + override def getLinkedAccount(implicit userInfo: UserInfo): Future[Option[LinkedEraAccount]] = Future.successful { + logger.warn("Getting Linked eRA Account from ECM, but ECM is disabled.") + None + } + + override def putLinkedEraAccount(linkedEraAccount: LinkedEraAccount)(implicit orchInfo: WithAccessToken): Future[Unit] = Future.successful { + logger.warn("Putting Linked eRA Account to ECM, but ECM is disabled.") + } + + override def deleteLinkedEraAccount(userInfo: UserInfo)(implicit orchInfo: WithAccessToken): Future[Unit] = Future.successful { + logger.warn("Deleting Linked eRA Account from ECM, but ECM is disabled.") + } + + override def getLinkedEraAccountForUsername(username: String)(implicit orchInfo: WithAccessToken): Future[Option[LinkedEraAccount]] = Future.successful { + logger.warn("Getting Linked eRA Account for username from ECM, but ECM is disabled.") + None + } + + override def getActiveLinkedEraAccounts(implicit orchInfo: WithAccessToken): Future[Seq[LinkedEraAccount]] = Future.successful { + logger.warn("Getting Active Linked eRA Accounts from ECM, but ECM is disabled.") + Seq.empty + } +} diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ExternalCredsDAO.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ExternalCredsDAO.scala new file mode 100644 index 000000000..401803d89 --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/ExternalCredsDAO.scala @@ -0,0 +1,25 @@ +package org.broadinstitute.dsde.firecloud.dataaccess + +import org.broadinstitute.dsde.firecloud.model.{LinkedEraAccount, UserInfo, WithAccessToken} +import org.databiosphere.workspacedata.client.ApiException + +import scala.concurrent.Future + +trait ExternalCredsDAO { + + @throws(classOf[ApiException]) + def getLinkedAccount(implicit userInfo: UserInfo): Future[Option[LinkedEraAccount]] + + @throws(classOf[ApiException]) + def putLinkedEraAccount(linkedEraAccount: LinkedEraAccount)(implicit orchInfo: WithAccessToken): Future[Unit] + + @throws(classOf[ApiException]) + def deleteLinkedEraAccount(userInfo: UserInfo)(implicit orchInfo: WithAccessToken): Future[Unit] + + @throws(classOf[ApiException]) + def getLinkedEraAccountForUsername(username: String)(implicit orchInfo: WithAccessToken): Future[Option[LinkedEraAccount]] + + @throws(classOf[ApiException]) + def getActiveLinkedEraAccounts(implicit orchInfo: WithAccessToken): Future[Seq[LinkedEraAccount]] + +} diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpExternalCredsDAO.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpExternalCredsDAO.scala new file mode 100644 index 000000000..76151d0b7 --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpExternalCredsDAO.scala @@ -0,0 +1,86 @@ +package org.broadinstitute.dsde.firecloud.dataaccess + +import bio.terra.externalcreds.api.OauthApi +import bio.terra.externalcreds.api.AdminApi +import bio.terra.externalcreds.client.ApiClient +import bio.terra.externalcreds.model.Provider +import com.google.api.client.http.HttpStatusCodes +import org.broadinstitute.dsde.firecloud.FireCloudConfig +import org.broadinstitute.dsde.firecloud.model.LinkedEraAccount.unapply +import org.broadinstitute.dsde.firecloud.model.{LinkedEraAccount, UserInfo, WithAccessToken} +import org.broadinstitute.dsde.workbench.model.WorkbenchException +import org.joda.time.DateTime +import org.springframework.web.client.{HttpClientErrorException, RestTemplate} + +import scala.concurrent.{ExecutionContext, Future} +import scala.jdk.CollectionConverters._ + +class HttpExternalCredsDAO(implicit val executionContext: ExecutionContext) extends ExternalCredsDAO { + + private lazy val restTemplate = new RestTemplate + + private def handleError[A](e: HttpClientErrorException, operation: String): Option[A] = { + e.getStatusCode.value() match { + case HttpStatusCodes.STATUS_CODE_NOT_FOUND => None + case _ => throw new WorkbenchException(s"Failed to $operation: ${e.getMessage}") + } + } + + override def getLinkedAccount(implicit userInfo: UserInfo): Future[Option[LinkedEraAccount]] = Future { + val oauthApi: OauthApi = getOauthApi(userInfo.accessToken.token) + try { + val linkInfo = oauthApi.getLink(Provider.ERA_COMMONS) + Some(LinkedEraAccount(userInfo.id, linkInfo.getExternalUserId, new DateTime(linkInfo.getExpirationTimestamp))) + } catch { + case e: HttpClientErrorException => handleError(e, "GET eRA Linked Account") + } + } + + override def putLinkedEraAccount(linkedEraAccount: LinkedEraAccount)(implicit orchInfo: WithAccessToken): Future[Unit] = Future { + val adminApi = getAdminApi(orchInfo.accessToken.token) + adminApi.putLinkedAccountWithFakeToken(unapply(linkedEraAccount), Provider.ERA_COMMONS) + } + + override def deleteLinkedEraAccount(userInfo: UserInfo)(implicit orchInfo: WithAccessToken): Future[Unit] = Future { + val adminApi = getAdminApi(orchInfo.accessToken.token) + try { + adminApi.adminDeleteLinkedAccount(userInfo.id, Provider.ERA_COMMONS) + } catch { + case e: HttpClientErrorException => handleError(e, "DELETE eRA Linked Account") + } + } + + override def getLinkedEraAccountForUsername(username: String)(implicit orchInfo: WithAccessToken): Future[Option[LinkedEraAccount]] = Future { + val adminApi = getAdminApi(orchInfo.accessToken.token) + try { + val adminLinkInfo = adminApi.getLinkedAccountForExternalId(Provider.ERA_COMMONS, username) + Some(LinkedEraAccount(adminLinkInfo)) + } catch { + case e: HttpClientErrorException => handleError(e, s"GET eRA Linked Account for username [$username]") + } + } + + override def getActiveLinkedEraAccounts(implicit orchInfo: WithAccessToken): Future[Seq[LinkedEraAccount]] = Future { + val adminApi = getAdminApi(orchInfo.accessToken.token) + val adminLinkInfos = adminApi.getActiveLinkedAccounts(Provider.ERA_COMMONS) + adminLinkInfos.asScala.map(LinkedEraAccount.apply).toSeq + } + + private def getApi(accessToken: String): ApiClient = { + val client = new ApiClient(restTemplate) + client.setBasePath(FireCloudConfig.ExternalCreds.baseUrl) + client.setAccessToken(accessToken) + client + } + + private def getOauthApi(accessToken: String): OauthApi = { + val client = getApi(accessToken) + new OauthApi(client) + } + + private def getAdminApi(accessToken: String): AdminApi = { + val client = getApi(accessToken) + new AdminApi(client) + } + +} diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpSamDAO.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpSamDAO.scala index f257ffc11..f84b26ad4 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpSamDAO.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/HttpSamDAO.scala @@ -12,12 +12,12 @@ import org.broadinstitute.dsde.firecloud.model.ErrorReportExtensions.FCErrorRepo import org.broadinstitute.dsde.firecloud.model.ManagedGroupRoles.ManagedGroupRole import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ import org.broadinstitute.dsde.firecloud.model.SamResource.UserPolicy -import org.broadinstitute.dsde.firecloud.model.{AccessToken, FireCloudManagedGroupMembership, ManagedGroupRoles, RegistrationInfo, RegistrationInfoV2, SamUserAttributesRequest, SamUserRegistrationRequest, SamUserResponse, UserIdInfo, UserInfo, WithAccessToken} +import org.broadinstitute.dsde.firecloud.model.{AccessToken, FireCloudManagedGroupMembership, ManagedGroupRoles, RegistrationInfo, SamUser, SamUserAttributesRequest, SamUserRegistrationRequest, SamUserResponse, UserIdInfo, UserInfo, WithAccessToken, WorkbenchUserInfo} import org.broadinstitute.dsde.firecloud.utils.RestJsonClient import org.broadinstitute.dsde.rawls.model.RawlsUserEmail import org.broadinstitute.dsde.workbench.model.WorkbenchIdentityJsonSupport._ import org.broadinstitute.dsde.workbench.model.google.GoogleProject -import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} +import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName, WorkbenchUserId} import org.broadinstitute.dsde.workbench.util.health.SubsystemStatus import spray.json.DefaultJsonProtocol._ import spray.json.{JsValue, JsonFormat, RootJsonFormat} @@ -50,6 +50,14 @@ class HttpSamDAO( implicit val system: ActorSystem, val materializer: Materializ authedRequestToObject[UserIdInfo](Get(samGetUserIdsUrl.format(URLEncoder.encode(email.value, UTF_8.name)))) } + // Sam's API only allows for 1000 user to be fetched at one time + override def getUsersForIds(samUserIds: Seq[WorkbenchUserId])(implicit userInfo: WithAccessToken): Future[Seq[WorkbenchUserInfo]] = Future.sequence { + samUserIds.sliding(1000, 1000).toSeq.map { batch => + adminAuthedRequestToObject[Seq[SamUser]](Post(samAdminGetUsersForIdsUrl, batch)) + .map(_.map(user => WorkbenchUserInfo(user.id.value, user.email.value))) + } + }.map(_.flatten) + override def isGroupMember(groupName: WorkbenchGroupName, userInfo: UserInfo): Future[Boolean] = { implicit val accessToken = userInfo authedRequestToObject[List[String]](Get(samResourceRoles(managedGroupResourceTypeName, groupName.value)), label=Some("HttpSamDAO.isGroupMember")).map { allRoles => diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/SamDAO.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/SamDAO.scala index e25661e16..ba647ffad 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/SamDAO.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/dataaccess/SamDAO.scala @@ -6,10 +6,10 @@ import com.typesafe.scalalogging.LazyLogging import org.broadinstitute.dsde.firecloud.FireCloudConfig import org.broadinstitute.dsde.firecloud.model.ManagedGroupRoles.ManagedGroupRole import org.broadinstitute.dsde.firecloud.model.SamResource.UserPolicy -import org.broadinstitute.dsde.firecloud.model.{AccessToken, FireCloudManagedGroupMembership, RegistrationInfo, RegistrationInfoV2, SamUserResponse, UserIdInfo, UserInfo, WithAccessToken} +import org.broadinstitute.dsde.firecloud.model.{AccessToken, FireCloudManagedGroupMembership, RegistrationInfo, RegistrationInfoV2, SamUserResponse, UserIdInfo, UserInfo, WithAccessToken, WorkbenchUserInfo} import org.broadinstitute.dsde.rawls.model.{ErrorReportSource, RawlsUserEmail} import org.broadinstitute.dsde.workbench.model.google.GoogleProject -import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} +import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName, WorkbenchUserId} import org.broadinstitute.dsde.workbench.util.health.Subsystems import scala.concurrent.Future @@ -35,6 +35,7 @@ trait SamDAO extends LazyLogging with ReportsSubsystemStatus { val samGetUserIdsUrl = FireCloudConfig.Sam.baseUrl + "/api/users/v1/%s" val samArbitraryPetTokenUrl = FireCloudConfig.Sam.baseUrl + "/api/google/v1/user/petServiceAccount/token" val samPetKeyForProject = FireCloudConfig.Sam.baseUrl + "/api/google/v1/user/petServiceAccount/%s/key" + val samAdminGetUsersForIdsUrl = FireCloudConfig.Sam.baseUrl + "/api/admin/v2/users" val samManagedGroupsBase: String = FireCloudConfig.Sam.baseUrl + "/api/groups" val samManagedGroupBase: String = FireCloudConfig.Sam.baseUrl + "/api/group" @@ -61,6 +62,8 @@ trait SamDAO extends LazyLogging with ReportsSubsystemStatus { def getUserIds(email: RawlsUserEmail)(implicit userInfo: WithAccessToken): Future[UserIdInfo] + def getUsersForIds(samUserIds: Seq[WorkbenchUserId])(implicit userInfo: WithAccessToken): Future[Seq[WorkbenchUserInfo]] + def listWorkspaceResources(implicit userInfo: WithAccessToken): Future[Seq[UserPolicy]] def createGroup(groupName: WorkbenchGroupName)(implicit userInfo: WithAccessToken): Future[Unit] diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/model/LinkedEraAccount.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/model/LinkedEraAccount.scala new file mode 100644 index 000000000..b31c78d2a --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/model/LinkedEraAccount.scala @@ -0,0 +1,25 @@ +package org.broadinstitute.dsde.firecloud.model + +import bio.terra.externalcreds.model.AdminLinkInfo +import org.joda.time.{DateTime, Instant} + +object LinkedEraAccount { + def apply(samUserId: String, nihLink: NihLink): LinkedEraAccount = { + LinkedEraAccount(samUserId, nihLink.linkedNihUsername, Instant.ofEpochSecond(nihLink.linkExpireTime).toDateTime) + } + + def apply(adminLinkInfo: AdminLinkInfo): LinkedEraAccount = { + LinkedEraAccount(adminLinkInfo.getUserId, adminLinkInfo.getLinkedExternalId, new DateTime(adminLinkInfo.getLinkExpireTime)) + } + + def unapply(linkedEraAccount: LinkedEraAccount): AdminLinkInfo = { + new AdminLinkInfo() + .userId(linkedEraAccount.userId) + .linkedExternalId(linkedEraAccount.linkedExternalId) + .linkExpireTime(linkedEraAccount.linkExpireTime.toDate) + } +} + +case class LinkedEraAccount(userId: String, linkedExternalId: String, linkExpireTime: DateTime) + + diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala index cfae7d634..907d9ff03 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/model/ModelJsonProtocol.scala @@ -16,6 +16,7 @@ import org.broadinstitute.dsde.firecloud.model.ShareLog.{Share, ShareType} import org.broadinstitute.dsde.rawls.model.UserModelJsonSupport._ import org.broadinstitute.dsde.rawls.model.WorkspaceACLJsonSupport.WorkspaceAccessLevelFormat import org.broadinstitute.dsde.rawls.model._ +import org.broadinstitute.dsde.workbench.client.sam.model.User import org.broadinstitute.dsde.workbench.model.ValueObjectFormat import org.broadinstitute.dsde.workbench.model.WorkbenchIdentityJsonSupport._ import org.broadinstitute.dsde.workbench.model.google.GoogleModelJsonSupport.InstantFormat @@ -233,6 +234,7 @@ object ModelJsonProtocol extends WorkspaceJsonSupport with SprayJsonSupport { implicit val impRegistrationInfo: RootJsonFormat[RegistrationInfo] = jsonFormat3(RegistrationInfo) implicit val impRegistrationInfoV2: RootJsonFormat[RegistrationInfoV2] = jsonFormat3(RegistrationInfoV2) implicit val impSamUserResponse: RootJsonFormat[SamUserResponse] = jsonFormat8(SamUserResponse) + implicit val impSamUser: RootJsonFormat[SamUser] = jsonFormat8(SamUser) implicit val impUserIdInfo: RootJsonFormat[UserIdInfo] = jsonFormat3(UserIdInfo) implicit val impCurator: RootJsonFormat[Curator] = jsonFormat1(Curator) implicit val impUserImportPermission: RootJsonFormat[UserImportPermission] = jsonFormat2(UserImportPermission) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/model/Profile.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/model/Profile.scala index 1f595a55c..174146cfa 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/model/Profile.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/model/Profile.scala @@ -107,6 +107,10 @@ object Profile { } +object NihLink { + def apply(linkedEraAccount: LinkedEraAccount): NihLink = NihLink(linkedEraAccount.linkedExternalId, linkedEraAccount.linkExpireTime.getMillis / 1000) +} + case class NihLink(linkedNihUsername: String, linkExpireTime: Long) extends mappedPropVals { require(ProfileValidator.nonEmpty(linkedNihUsername), "linkedNihUsername must be non-empty") } diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUser.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUser.scala new file mode 100644 index 000000000..f3e4333aa --- /dev/null +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/model/SamUser.scala @@ -0,0 +1,14 @@ +package org.broadinstitute.dsde.firecloud.model + +import org.broadinstitute.dsde.workbench.model.{AzureB2CId, GoogleSubjectId, WorkbenchEmail, WorkbenchUserId} + +import java.time.Instant + +case class SamUser(id: WorkbenchUserId, + googleSubjectId: Option[GoogleSubjectId], + email: WorkbenchEmail, + azureB2CId: Option[AzureB2CId], + enabled: Boolean, + createdAt: Instant, + registeredAt: Option[Instant], + updatedAt: Instant) diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/service/NihService.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/service/NihService.scala index 2f4b726e0..dac4c6417 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/service/NihService.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/service/NihService.scala @@ -4,14 +4,14 @@ import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport import akka.http.scaladsl.model.StatusCodes import akka.http.scaladsl.model.StatusCodes._ import com.typesafe.scalalogging.LazyLogging -import org.broadinstitute.dsde.firecloud.dataaccess.{GoogleServicesDAO, SamDAO, ShibbolethDAO, ThurloeDAO} +import org.broadinstitute.dsde.firecloud.dataaccess.{ExternalCredsDAO, GoogleServicesDAO, SamDAO, ShibbolethDAO, ThurloeDAO} import org.broadinstitute.dsde.firecloud.model.ModelJsonProtocol._ import org.broadinstitute.dsde.firecloud.model._ import org.broadinstitute.dsde.firecloud.service.PerRequest.{PerRequestMessage, RequestComplete} import org.broadinstitute.dsde.firecloud.utils.DateUtils import org.broadinstitute.dsde.firecloud.{Application, FireCloudConfig, FireCloudException, FireCloudExceptionWithErrorReport} import org.broadinstitute.dsde.rawls.model.ErrorReport -import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName} +import org.broadinstitute.dsde.workbench.model.{WorkbenchEmail, WorkbenchGroupName, WorkbenchUserId} import org.slf4j.LoggerFactory import pdi.jwt.{Jwt, JwtAlgorithm} import spray.json.DefaultJsonProtocol._ @@ -19,8 +19,7 @@ import spray.json._ import scala.concurrent.{ExecutionContext, Future} import scala.io.Source -import scala.reflect.runtime.universe.{MethodSymbol, typeOf} -import scala.util.{Failure, Try} +import scala.util.{Failure, Success, Try} case class NihStatus( @@ -28,7 +27,7 @@ case class NihStatus( datasetPermissions: Set[NihDatasetPermission], linkExpireTime: Option[Long] = None) -case class NihWhitelist( +case class NihAllowlist( name: String, groupToSync: WorkbenchGroupName, fileName: String) @@ -42,93 +41,158 @@ object NihStatus { object NihService { def constructor(app: Application)()(implicit executionContext: ExecutionContext) = - new NihService(app.samDAO, app.thurloeDAO, app.googleServicesDAO, app.shibbolethDAO) + new NihService(app.samDAO, app.thurloeDAO, app.googleServicesDAO, app.shibbolethDAO, app.ecmDAO) } -class NihService(val samDao: SamDAO, val thurloeDao: ThurloeDAO, val googleDao: GoogleServicesDAO, val shibbolethDao: ShibbolethDAO) +class NihService(val samDao: SamDAO, val thurloeDao: ThurloeDAO, val googleDao: GoogleServicesDAO, val shibbolethDao: ShibbolethDAO, val ecmDao: ExternalCredsDAO) (implicit val executionContext: ExecutionContext) extends LazyLogging with SprayJsonSupport { lazy val log = LoggerFactory.getLogger(getClass) def getAdminAccessToken: WithAccessToken = UserInfo(googleDao.getAdminUserAccessToken, "") - val nihWhitelists: Set[NihWhitelist] = FireCloudConfig.Nih.whitelists + private val nihAllowlists: Set[NihAllowlist] = FireCloudConfig.Nih.whitelists def getNihStatus(userInfo: UserInfo): Future[PerRequestMessage] = { + getNihStatusFromEcm(userInfo).flatMap { + case Some(nihStatus) => + logger.info("Found eRA Commons link in ECM for user " + userInfo.id) + Future.successful(RequestComplete(nihStatus)) + case None => getNihStatusFromThurloe(userInfo).map { + case Some(nihStatus) => + logger.info("Found eRA Commons link in Thurloe for user " + userInfo.id) + RequestComplete(nihStatus) + case None => RequestComplete(NotFound) + } + } + } + + private def getNihStatusFromEcm(userInfo: UserInfo): Future[Option[NihStatus]] = { + ecmDao.getLinkedAccount(userInfo).flatMap { + case Some(linkedAccount) => getAllAllowlistGroupMemberships(userInfo).map { allowlistMembership => + Some(NihStatus(Some(linkedAccount.linkedExternalId), allowlistMembership, Some(linkedAccount.linkExpireTime.getMillis / 1000L))) + } + case None => Future.successful(None) + } + } + + private def getNihStatusFromThurloe(userInfo: UserInfo): Future[Option[NihStatus]] = { thurloeDao.getAllKVPs(userInfo.id, userInfo) flatMap { case Some(profileWrapper) => ProfileUtils.getString("linkedNihUsername", profileWrapper) match { - case Some(linkedNihUsername) => - Future.traverse(nihWhitelists) { whitelistDef => - samDao.isGroupMember(whitelistDef.groupToSync, userInfo).map(isMember => NihDatasetPermission(whitelistDef.name, isMember)) - }.map { whitelistMembership => + case Some(linkedNihUsername) => getAllAllowlistGroupMemberships(userInfo).map { allowlistMembership => val linkExpireTime = ProfileUtils.getLong("linkExpireTime", profileWrapper) - RequestComplete(NihStatus(Some(linkedNihUsername), whitelistMembership, linkExpireTime)) + Some(NihStatus(Some(linkedNihUsername), allowlistMembership, linkExpireTime)) } - case None => Future.successful(RequestComplete(NotFound)) + case None => Future.successful(None) } - case None => Future.successful(RequestComplete(NotFound)) - } recover { - case e:Exception => RequestCompleteWithErrorReport(InternalServerError, e.getMessage, e) + case None => Future.successful(None) + } + } + + private def getAllAllowlistGroupMemberships(userInfo: UserInfo): Future[Set[NihDatasetPermission]] = { + val groupMemberships = samDao.listGroups(userInfo) + groupMemberships.map { groups => + val samGroupNames = groups.map(g => WorkbenchGroupName(g.groupName)).toSet + nihAllowlists.map(allowlist => NihDatasetPermission(allowlist.name, samGroupNames.contains(allowlist.groupToSync))) } } - private def downloadNihWhitelist(whitelist: NihWhitelist): Set[String] = { - val usersList = Source.fromInputStream(googleDao.getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, whitelist.fileName)) + private def downloadNihAllowlist(allowlist: NihAllowlist): Set[String] = { + val usersList = Source.fromInputStream(googleDao.getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, allowlist.fileName)) usersList.getLines().toSet } - def syncWhitelistAllUsers(whitelistName: String): Future[PerRequestMessage] = { - nihWhitelists.find(_.name.equals(whitelistName)) match { - case Some(whitelist) => - val whitelistSyncResults = syncNihWhitelistAllUsers(whitelist) - whitelistSyncResults map { _ => RequestComplete(NoContent) } + def syncAllowlistAllUsers(allowlistName: String): Future[PerRequestMessage] = { + logger.info("Synchronizing allowlist '" + allowlistName + "' for all users") + nihAllowlists.find(_.name.equals(allowlistName)) match { + case Some(allowlist) => + val allowlistSyncResults = syncNihAllowlistAllUsers(allowlist) + allowlistSyncResults map { _ => RequestComplete(NoContent) } case None => Future.successful(RequestComplete(NotFound)) } } - // This syncs all of the whitelists for all of the users - def syncAllNihWhitelistsAllUsers(): Future[PerRequestMessage] = { - val whitelistSyncResults = Future.traverse(nihWhitelists)(syncNihWhitelistAllUsers) + // This syncs all of the allowlists for all of the users + def syncAllNihAllowlistsAllUsers(): Future[PerRequestMessage] = { + logger.info("Synchronizing all allowlists for all users") + val allowlistSyncResults = Future.traverse(nihAllowlists)(syncNihAllowlistAllUsers) - whitelistSyncResults map { _ => RequestComplete(NoContent) } + allowlistSyncResults map { _ => RequestComplete(NoContent) } } - // This syncs the specified whitelist in full - def syncNihWhitelistAllUsers(nihWhitelist: NihWhitelist): Future[Unit] = { - val whitelistUsers = downloadNihWhitelist(nihWhitelist) - + private def getNihAllowlistTerraEmailsFromEcm(allowlistEraUsernames: Set[String]): Future[Set[WorkbenchEmail]] = + for { + // The list of users that, according to ECM, have active links + allLinkedAccounts <- ecmDao.getActiveLinkedEraAccounts(getAdminAccessToken) + // The list of linked accounts which for which the user appears in the allowlist + allowlistLinkedAccounts = allLinkedAccounts.filter(linkedAccount => allowlistEraUsernames.contains(linkedAccount.linkedExternalId)) + // The users from Sam for the linked accounts on the allowlist + users <- samDao.getUsersForIds(allowlistLinkedAccounts.map(la => WorkbenchUserId(la.userId)))(getAdminAccessToken) + } yield users.map(user => WorkbenchEmail(user.userEmail)).toSet + + private def getNihAllowlistTerraEmailsFromThurloe(allowlistEraUsernames: Set[String]): Future[Set[WorkbenchEmail]] = for { // The list of users that, according to Thurloe, have active links and are - // on the specified whitelist + // on the specified allowlist subjectIds <- getCurrentNihUsernameMap(thurloeDao) map { mapping => - mapping.collect { case (fcUser, nihUser) if whitelistUsers contains nihUser => fcUser }.toSeq + mapping.collect { case (fcUser, nihUser) if allowlistEraUsernames contains nihUser => fcUser }.toSeq } //Sam APIs don't consume subject IDs. Now we must look up the emails in Thurloe... members <- thurloeDao.getAllUserValuesForKey("email").map { keyValues => keyValues.view.filterKeys(subjectId => subjectIds.contains(subjectId)).values.map(WorkbenchEmail).toList } + } yield members.toSet - _ <- ensureWhitelistGroupsExists() + // This syncs the specified allowlist in full + private def syncNihAllowlistAllUsers(nihAllowlist: NihAllowlist): Future[Unit] = { + val allowlistUsers = downloadNihAllowlist(nihAllowlist) - // The request to rawls to completely overwrite the group - // with the list of actively linked users on the whitelist - _ <- samDao.overwriteGroupMembers(nihWhitelist.groupToSync, ManagedGroupRoles.Member, members)(getAdminAccessToken) recoverWith { - case e: Exception => throw new FireCloudException(s"Error synchronizing NIH whitelist: ${e.getMessage}") + for { + ecmEmails <- getNihAllowlistTerraEmailsFromEcm(allowlistUsers) + thurloeEmails <- getNihAllowlistTerraEmailsFromThurloe(allowlistUsers) + members = ecmEmails ++ thurloeEmails + _ <- ensureAllowlistGroupsExists() + // The request to Sam to completely overwrite the group with the list of actively linked users on the allowlist + _ <- samDao.overwriteGroupMembers(nihAllowlist.groupToSync, ManagedGroupRoles.Member, members.toList)(getAdminAccessToken) recoverWith { + case e: Exception => throw new FireCloudException(s"Error synchronizing NIH allowlist: ${e.getMessage}") } } yield () } - private def linkNihAccount(userInfo: UserInfo, nihLink: NihLink): Future[Try[Unit]] = { + private def linkNihAccountEcm(userInfo: UserInfo, nihLink: NihLink): Future[Try[Unit]] = { + ecmDao.putLinkedEraAccount(LinkedEraAccount(userInfo.id, nihLink))(getAdminAccessToken) + .flatMap(_ => { + logger.info("Successfully linked NIH account in ECM for user " + userInfo.id) + Future.successful(Success()) + }).recoverWith { + case e => + logger.warn("Failed to link NIH account in ECM for user" + userInfo.id) + Future.successful(Failure(e)) + } + } + + private def linkNihAccountThurloe(userInfo: UserInfo, nihLink: NihLink): Future[Try[Unit]] = { val profilePropertyMap = nihLink.propertyValueMap thurloeDao.saveKeyValues(userInfo, profilePropertyMap) } private def unlinkNihAccount(userInfo: UserInfo): Future[Unit] = { + for { + _ <- unlinkNihAccountEcm(userInfo) + _ <- unlinkNihAccountThurloe(userInfo) + } yield () + } + + private def unlinkNihAccountEcm(userInfo: UserInfo): Future[Unit] = { + ecmDao.deleteLinkedEraAccount(userInfo)(getAdminAccessToken) + } + + private def unlinkNihAccountThurloe(userInfo: UserInfo): Future[Unit] = { val nihKeys = Set("linkedNihUsername", "linkExpireTime") Future.traverse(nihKeys) { nihKey => @@ -147,9 +211,9 @@ class NihService(val samDao: SamDAO, val thurloeDao: ThurloeDAO, val googleDao: def unlinkNihAccountAndSyncSelf(userInfo: UserInfo): Future[Unit] = { for { _ <- unlinkNihAccount(userInfo) - _ <- ensureWhitelistGroupsExists() - _ <- Future.traverse(nihWhitelists) { - whitelist => removeUserFromNihWhitelistGroup(WorkbenchEmail(userInfo.userEmail), whitelist).recoverWith { + _ <- ensureAllowlistGroupsExists() + _ <- Future.traverse(nihAllowlists) { + allowlist => removeUserFromNihAllowlistGroup(WorkbenchEmail(userInfo.userEmail), allowlist).recoverWith { case _: Exception => throw new FireCloudExceptionWithErrorReport(ErrorReport(StatusCodes.InternalServerError, "Unable to unlink NIH account")) } } @@ -169,16 +233,23 @@ class NihService(val samDao: SamDAO, val thurloeDao: ThurloeDAO, val googleDao: case (_, rawTokenFromShibboleth, _) => rawTokenFromShibboleth.parseJson.convertTo[ShibbolethToken].toNihLink } - linkResult <- linkNihAccount(userInfo, nihLink) - _ <- ensureWhitelistGroupsExists() - whitelistSyncResults <- Future.traverse(nihWhitelists) { - whitelist => syncNihWhitelistForUser(WorkbenchEmail(userInfo.userEmail), nihLink.linkedNihUsername, whitelist) - .map(NihDatasetPermission(whitelist.name, _)) + thurloeLinkResult <- linkNihAccountThurloe(userInfo, nihLink) + ecmLinkResult <- linkNihAccountEcm(userInfo, nihLink) + + _ <- ensureAllowlistGroupsExists() + allowlistSyncResults <- Future.traverse(nihAllowlists) { + allowlist => syncNihAllowlistForUser(WorkbenchEmail(userInfo.userEmail), nihLink.linkedNihUsername, allowlist) + .map(NihDatasetPermission(allowlist.name, _)) } } yield { - if (linkResult.isSuccess) { - RequestComplete(OK, NihStatus(Option(nihLink.linkedNihUsername), whitelistSyncResults, Option(nihLink.linkExpireTime))) + if (thurloeLinkResult.isSuccess && ecmLinkResult.isSuccess) { + RequestComplete(OK, NihStatus(Option(nihLink.linkedNihUsername), allowlistSyncResults, Option(nihLink.linkExpireTime))) } else { + (thurloeLinkResult, ecmLinkResult) match { + case (Failure(t), Success(_)) => logger.error("Failed to link NIH Account in Thurloe", t) + case (Success(_), Failure(t)) => logger.error("Failed to link NIH Account in ECM", t) + case (Failure(t1), Failure(t2)) => logger.error("Failed to link NIH Account in Thurloe and ECM", t1, t2) + } RequestCompleteWithErrorReport(InternalServerError, "Error updating NIH link") } } @@ -189,27 +260,27 @@ class NihService(val samDao: SamDAO, val thurloeDao: ThurloeDAO, val googleDao: } } - private def syncNihWhitelistForUser(userEmail: WorkbenchEmail, linkedNihUserName: String, nihWhitelist: NihWhitelist): Future[Boolean] = { - val whitelistUsers = downloadNihWhitelist(nihWhitelist) + private def syncNihAllowlistForUser(userEmail: WorkbenchEmail, linkedNihUserName: String, nihAllowlist: NihAllowlist): Future[Boolean] = { + val allowlistUsers = downloadNihAllowlist(nihAllowlist) - if(whitelistUsers contains linkedNihUserName) { + if(allowlistUsers contains linkedNihUserName) { for { - _ <- samDao.addGroupMember(nihWhitelist.groupToSync, ManagedGroupRoles.Member, userEmail)(getAdminAccessToken) + _ <- samDao.addGroupMember(nihAllowlist.groupToSync, ManagedGroupRoles.Member, userEmail)(getAdminAccessToken) } yield true } else { for { - _ <- samDao.removeGroupMember(nihWhitelist.groupToSync, ManagedGroupRoles.Member, userEmail)(getAdminAccessToken) + _ <- samDao.removeGroupMember(nihAllowlist.groupToSync, ManagedGroupRoles.Member, userEmail)(getAdminAccessToken) } yield false } } - private def removeUserFromNihWhitelistGroup(userEmail: WorkbenchEmail, nihWhitelist: NihWhitelist): Future[Unit] = { - samDao.removeGroupMember(nihWhitelist.groupToSync, ManagedGroupRoles.Member, userEmail)(getAdminAccessToken) + private def removeUserFromNihAllowlistGroup(userEmail: WorkbenchEmail, nihAllowlist: NihAllowlist): Future[Unit] = { + samDao.removeGroupMember(nihAllowlist.groupToSync, ManagedGroupRoles.Member, userEmail)(getAdminAccessToken) } - private def ensureWhitelistGroupsExists(): Future[Unit] = { + private def ensureAllowlistGroupsExists(): Future[Unit] = { samDao.listGroups(getAdminAccessToken).flatMap { groups => - val missingGroupNames = nihWhitelists.map(_.groupToSync.value.toLowerCase()) -- groups.map(_.groupName.toLowerCase).toSet + val missingGroupNames = nihAllowlists.map(_.groupToSync.value.toLowerCase()) -- groups.map(_.groupName.toLowerCase).toSet if (missingGroupNames.isEmpty) { Future.successful(()) } else { @@ -233,7 +304,7 @@ class NihService(val samDao: SamDAO, val thurloeDao: ThurloeDAO, val googleDao: } // get a mapping of FireCloud user name to NIH User name, for only those Thurloe users with a non-expired NIH link - def getCurrentNihUsernameMap(thurloeDAO: ThurloeDAO): Future[Map[String, String]] = { + private def getCurrentNihUsernameMap(thurloeDAO: ThurloeDAO): Future[Map[String, String]] = { val nihUsernames = thurloeDAO.getAllUserValuesForKey("linkedNihUsername") val nihExpireTimes = thurloeDAO.getAllUserValuesForKey("linkExpireTime") diff --git a/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiService.scala b/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiService.scala index 81ce5c8d0..7dd893ca9 100644 --- a/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiService.scala +++ b/src/main/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiService.scala @@ -21,11 +21,11 @@ trait NihApiService extends Directives with RequestBuilding with EnabledUserDire val syncRoute: Route = path("sync_whitelist" / Segment) { whitelistName => post { - complete { nihServiceConstructor().syncWhitelistAllUsers(whitelistName) } + complete { nihServiceConstructor().syncAllowlistAllUsers(whitelistName) } } } ~ path("sync_whitelist") { post { - complete { nihServiceConstructor().syncAllNihWhitelistsAllUsers() } + complete { nihServiceConstructor().syncAllNihAllowlistsAllUsers() } } } diff --git a/src/test/resources/reference.conf b/src/test/resources/reference.conf index 4cc76b642..f3d0d4267 100644 --- a/src/test/resources/reference.conf +++ b/src/test/resources/reference.conf @@ -43,6 +43,11 @@ cwds { bucketName = "cwds-testconf-bucketname" } +externalCreds { + baseUrl = "https://externalcreds.dsde-dev.broadinstitute.org" + enabled = true +} + firecloud { baseUrl = "https://local.broadinstitute.org" portalUrl = "https://local.broadinstitute.org" diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockSamDAO.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockSamDAO.scala index 5ee1b8ce0..9945a5740 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockSamDAO.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/dataaccess/MockSamDAO.scala @@ -142,4 +142,6 @@ class MockSamDAO extends SamDAO { Future.failed(new FireCloudExceptionWithErrorReport(new ErrorReport("sam", "invalid", Some(StatusCodes.BadRequest), Seq.empty, Seq.empty, None))) } } + + override def getUsersForIds(samUserIds: Seq[WorkbenchUserId])(implicit userInfo: WithAccessToken): Future[Seq[WorkbenchUserInfo]] = Future.successful(Seq()) } diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/service/BaseServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/service/BaseServiceSpec.scala index 8b444fef5..6eed89476 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/service/BaseServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/service/BaseServiceSpec.scala @@ -22,8 +22,9 @@ class BaseServiceSpec extends ServiceSpec with BeforeAndAfter { val shareLogDao:MockShareLogDAO = new MockShareLogDAO val shibbolethDao:MockShibbolethDAO = new MockShibbolethDAO val cwdsDao:CwdsDAO = new MockCwdsDAO + val ecmDao:ExternalCredsDAO = new DisabledExternalCredsDAO val app:Application = - new Application(agoraDao, googleServicesDao, ontologyDao, rawlsDao, samDao, searchDao, researchPurposeSupport, thurloeDao, shareLogDao, shibbolethDao, cwdsDao) + new Application(agoraDao, googleServicesDao, ontologyDao, rawlsDao, samDao, searchDao, researchPurposeSupport, thurloeDao, shareLogDao, shibbolethDao, cwdsDao, ecmDao) } diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceSpec.scala index 483a6f331..371c26558 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceSpec.scala @@ -29,9 +29,10 @@ class NihServiceSpec extends AnyFlatSpec with Matchers { val thurloeDao = new MockThurloeDAO val googleDao = new MockGoogleServicesDAO val shibbolethDao = new MockShibbolethDAO + val ecmDao = new DisabledExternalCredsDAO // build the service instance we'll use for tests - val nihService = new NihService(samDao, thurloeDao, googleDao, shibbolethDao) + val nihService = new NihService(samDao, thurloeDao, googleDao, shibbolethDao, ecmDao) val usernames = Map("fcSubjectId1" -> "nihUsername1", "fcSubjectId2" -> "nihUsername2") @@ -71,7 +72,7 @@ class NihServiceSpec extends AnyFlatSpec with Matchers { val mockShibboleth = mock[ShibbolethDAO] when(mockShibboleth.getPublicKey()).thenReturn(Future.successful(pubKey)) - val nihServiceMock = new NihService(samDao, thurloeDao, googleDao, mockShibboleth) + val nihServiceMock = new NihService(samDao, thurloeDao, googleDao, mockShibboleth, ecmDao) // expires in 15 minutes val expiresInTheFuture: Long = Instant.ofEpochMilli(System.currentTimeMillis() + (15 * 60 * 1000)).getEpochSecond // 15 minutes * 60 seconds * 1000 milliseconds diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceUnitSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceUnitSpec.scala new file mode 100644 index 000000000..ecfc8b0f9 --- /dev/null +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/service/NihServiceUnitSpec.scala @@ -0,0 +1,478 @@ +package org.broadinstitute.dsde.firecloud.service + +import akka.http.scaladsl.model.headers.OAuth2BearerToken +import akka.http.scaladsl.model.{StatusCode, StatusCodes} +import org.broadinstitute.dsde.firecloud.FireCloudConfig +import org.broadinstitute.dsde.firecloud.FireCloudException +import org.broadinstitute.dsde.firecloud.dataaccess.{ExternalCredsDAO, GoogleServicesDAO, SamDAO, ShibbolethDAO, ThurloeDAO} +import org.broadinstitute.dsde.firecloud.model.{FireCloudKeyValue, FireCloudManagedGroupMembership, JWTWrapper, LinkedEraAccount, ManagedGroupRoles, NihLink, ProfileWrapper, SamUser, UserInfo, WithAccessToken, WorkbenchUserInfo} +import org.broadinstitute.dsde.workbench.model.{AzureB2CId, GoogleSubjectId, WorkbenchEmail, WorkbenchGroupName, WorkbenchUserId} +import org.broadinstitute.dsde.rawls.model.ErrorReport +import org.joda.time.DateTime +import org.mockito.{ArgumentMatchers, Mockito} +import org.mockito.ArgumentMatchers.any +import org.mockito.Mockito.{never, times, verify, verifyNoInteractions, when} +import org.scalatest.BeforeAndAfterEach +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers +import org.scalatestplus.mockito.MockitoSugar.mock +import pdi.jwt.{Jwt, JwtAlgorithm, JwtClaim} + +import java.io.ByteArrayInputStream +import java.nio.charset.StandardCharsets +import java.security.{KeyPairGenerator, PrivateKey} +import java.time.Instant +import java.util.{Base64, UUID} +import scala.concurrent.duration.Duration +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.util.{Failure, Random, Success} + +class NihServiceUnitSpec extends AnyFlatSpec with Matchers with BeforeAndAfterEach { + + implicit val executionContext: ExecutionContext = scala.concurrent.ExecutionContext.Implicits.global + val samDao = mock[SamDAO] + val thurloeDao = mock[ThurloeDAO] + val googleDao = mock[GoogleServicesDAO] + val shibbolethDao = mock[ShibbolethDAO] + val ecmDao = mock[ExternalCredsDAO] + + // build the service instance we'll use for tests + val nihService = new NihService(samDao, thurloeDao, googleDao, shibbolethDao, ecmDao) + + val userNoLinkedAccount = genSamUser(); + val userNoAllowlists = genSamUser() + val userTcgaAndTarget = genSamUser(); + val userTcgaOnly = genSamUser(); + val userTargetOnly = genSamUser(); + + var userNoAllowlistsLinkedAccount = LinkedEraAccount(userNoAllowlists.id.value, "nihUsername1", new DateTime().plusDays(30)) + var userTcgaAndTargetLinkedAccount = LinkedEraAccount(userTcgaAndTarget.id.value, "nihUsername2", new DateTime().plusDays(30)) + var userTcgaOnlyLinkedAccount = LinkedEraAccount(userTcgaOnly.id.value, "nihUsername3", new DateTime().plusDays(30)) + var userTargetOnlyLinkedAccount = LinkedEraAccount(userTargetOnly.id.value, "nihUsername4", new DateTime().plusDays(30)) + + val samUsers = Seq(userNoLinkedAccount, userNoAllowlists, userTcgaAndTarget, userTcgaOnly, userTargetOnly) + val linkedAccounts = Seq(userNoAllowlistsLinkedAccount, userTcgaAndTargetLinkedAccount, userTcgaOnlyLinkedAccount, userTargetOnlyLinkedAccount) + + val idToSamUser = samUsers.groupBy(_.id).view.mapValues(_.head).toMap + + val linkedAccountsBySamUserId = Map( + userNoAllowlists.id -> userNoAllowlistsLinkedAccount, + userTcgaAndTarget.id -> userTcgaAndTargetLinkedAccount, + userTcgaOnly.id -> userTcgaOnlyLinkedAccount, + userTargetOnly.id -> userTargetOnlyLinkedAccount + ) + + val linkedAccountsByExternalId = Map( + userNoAllowlistsLinkedAccount.linkedExternalId -> userNoAllowlistsLinkedAccount, + userTcgaAndTargetLinkedAccount.linkedExternalId -> userTcgaAndTargetLinkedAccount, + userTcgaOnlyLinkedAccount.linkedExternalId -> userTcgaOnlyLinkedAccount, + userTargetOnlyLinkedAccount.linkedExternalId -> userTargetOnlyLinkedAccount + ) + + val samUserToGroups = { + Map( + userNoLinkedAccount.id -> Set("other-group"), + userNoAllowlists.id -> Set("other-group"), + userTcgaAndTarget.id -> Set("TCGA-dbGaP-Authorized", "TARGET-dbGaP-Authorized", "other-group"), + userTcgaOnly.id -> Set("TCGA-dbGaP-Authorized", "other-group"), + userTargetOnly.id -> Set("TARGET-dbGaP-Authorized", "other-group") + ) + } + + val samGroupMemberships = { + Map( + "TCGA-dbGaP-Authorized" -> Set(userTcgaAndTarget.id, userTcgaOnly.id), + "TARGET-dbGaP-Authorized" -> Set(userTcgaAndTarget.id, userTargetOnly.id), + "this-doesnt-matter" -> Set.empty + ) + } + + val accessTokenToUser = { + Map( + UUID.randomUUID().toString -> userNoLinkedAccount.id, + UUID.randomUUID().toString -> userNoAllowlists.id, + UUID.randomUUID().toString -> userTcgaAndTarget.id, + UUID.randomUUID().toString -> userTcgaOnly.id, + UUID.randomUUID().toString -> userTargetOnly.id + ) + } + + val userToAccessToken = accessTokenToUser.map(_.swap) + val adminAccessToken = UUID.randomUUID().toString + + override def beforeEach(): Unit = { + Mockito.reset(thurloeDao, ecmDao, googleDao, samDao) + mockSamUsers() + mockGoogleServicesDAO() + } + + "getNihStatus" should "prefer ECM over Thurloe" in { + mockEcmUsers() + val user = userTcgaAndTarget + val userInfo = UserInfo(userToAccessToken(user.id), userTcgaAndTarget.id.value) + val nihStatus = Await.result(nihService.getNihStatus(userInfo), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[NihStatus]].response + nihStatus.linkedNihUsername shouldBe Some(linkedAccountsBySamUserId(WorkbenchUserId(userInfo.id)).linkedExternalId) + verifyNoInteractions(thurloeDao) + verify(ecmDao).getLinkedAccount(userInfo) + } + + it should "talk to Thurloe if nothing link is found in ECM" in { + mockThurloeUsers() + when(ecmDao.getLinkedAccount(any[UserInfo])).thenReturn(Future.successful(None)) + val user = userTcgaAndTarget + val userInfo = UserInfo(userToAccessToken(user.id), userTcgaAndTarget.id.value) + val nihStatus = Await.result(nihService.getNihStatus(userInfo), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[NihStatus]].response + nihStatus.linkedNihUsername shouldBe Some(linkedAccountsBySamUserId(WorkbenchUserId(userInfo.id)).linkedExternalId) + verify(thurloeDao).getAllKVPs(user.id.value, userInfo) + } + + it should "return None if no linked account is found" in { + when(thurloeDao.getAllKVPs(any[String], any[WithAccessToken])).thenReturn(Future.successful(None)) + when(ecmDao.getLinkedAccount(any[UserInfo])).thenReturn(Future.successful(None)) + val user = userNoLinkedAccount + val userInfo = UserInfo(userToAccessToken(user.id), userNoLinkedAccount.id.value) + val nihStatus = Await.result(nihService.getNihStatus(userInfo), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[StatusCode]].response + nihStatus should be(StatusCodes.NotFound) + } + + it should "return None if a user if found in Thurloe, but no linkedNihUsername exists" in { + when(thurloeDao.getAllKVPs(any[String], any[WithAccessToken])) + .thenReturn(Future.successful(Some(ProfileWrapper(userNoLinkedAccount.id.value, List(FireCloudKeyValue(Some("email"), Some(userNoLinkedAccount.email.value))))))) + when(ecmDao.getLinkedAccount(any[UserInfo])).thenReturn(Future.successful(None)) + val user = userNoLinkedAccount + val userInfo = UserInfo(userToAccessToken(user.id), userNoLinkedAccount.id.value) + val nihStatus = Await.result(nihService.getNihStatus(userInfo), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[StatusCode]].response + nihStatus should be(StatusCodes.NotFound) + } + + private def verifyTargetGroupSynced(): Unit = { + val emailsToSync = Set(WorkbenchEmail(userTcgaAndTarget.email.value), WorkbenchEmail(userTargetOnly.email.value)) + val nihStatus = Await.result(nihService.syncAllowlistAllUsers("TARGET"), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[StatusCode]].response + + nihStatus should be(StatusCodes.NoContent) + verify(googleDao, never()).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "tcga-whitelist.txt") + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "target-whitelist.txt") + verify(samDao, times(1)).overwriteGroupMembers( + ArgumentMatchers.eq(WorkbenchGroupName("TARGET-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.argThat((list: List[WorkbenchEmail]) => list.toSet.equals(emailsToSync)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, never()).overwriteGroupMembers( + ArgumentMatchers.eq(WorkbenchGroupName("other-group")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.argThat((list: List[WorkbenchEmail]) => list.toSet.equals(emailsToSync)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + } + + "syncWhitelistAllUsers" should "sync all users for a single allowlist from ECM" in { + mockEcmUsers() + when(thurloeDao.getAllUserValuesForKey(any[String])).thenReturn(Future.successful(Map.empty)) + + verifyTargetGroupSynced() + } + + it should "sync all users for a single allowlist from Thurloe" in { + mockThurloeUsers() + when(ecmDao.getActiveLinkedEraAccounts(any[UserInfo])).thenReturn(Future.successful(Seq.empty)) + + verifyTargetGroupSynced() + } + + it should "sync all users by combining responses from ECM and Thurloe if they contain the same users" in { + mockEcmUsers() + mockThurloeUsers() + + verifyTargetGroupSynced() + } + + it should "sync all users by combining responses from ECM and Thurloe if they contain different users" in { + when(ecmDao.getActiveLinkedEraAccounts(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenReturn(Future.successful(Seq(userTargetOnlyLinkedAccount))) + when(thurloeDao.getAllUserValuesForKey(ArgumentMatchers.eq("email"))) + .thenReturn(Future.successful(samUsers.filter(u => !u.id.equals(userTargetOnly.id)).map(user => user.id.value -> user.email.value).toMap)) + when(thurloeDao.getAllUserValuesForKey(ArgumentMatchers.eq("linkedNihUsername"))) + .thenReturn(Future.successful(linkedAccountsBySamUserId.removed(WorkbenchUserId(userTargetOnlyLinkedAccount.userId)).map(tup => (tup._1.value, tup._2.linkedExternalId)))) + when(thurloeDao.getAllUserValuesForKey(ArgumentMatchers.eq("linkExpireTime"))) + .thenReturn(Future.successful(linkedAccountsBySamUserId.removed(WorkbenchUserId(userTargetOnlyLinkedAccount.userId)).map(tup => (tup._1.value, (tup._2.linkExpireTime.getMillis / 1000L).toString)))) + + verifyTargetGroupSynced() + } + + it should "respond with NOT FOUND if no allowlist is found" in { + val nihStatus = Await.result(nihService.syncAllowlistAllUsers("NOT_FOUND"), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[StatusCode]].response + + nihStatus should be(StatusCodes.NotFound) + + } + + it should "recover from a Sam API Exception with a FirecloudException" in { + val errorMessage = "Oops :(" + Mockito.reset(samDao) + mockEcmUsers() + mockThurloeUsers() + when(samDao.getUsersForIds(any[Seq[WorkbenchUserId]])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenAnswer(args => { + val userIds = args.getArgument(0).asInstanceOf[Seq[WorkbenchUserId]] + Future.successful(samUsers.filter(user => userIds.contains(WorkbenchUserId(user.id.value))).map(user => WorkbenchUserInfo(user.id.value, user.email.value))) + }) + when(samDao.overwriteGroupMembers(any(), any(), any())(any())).thenReturn(Future.failed(new RuntimeException(errorMessage))) + when(samDao.listGroups(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenReturn(Future.successful(samGroupMemberships.keys.map(groupName => FireCloudManagedGroupMembership(groupName, groupName + "@firecloud.org", "member")).toList)) + when(samDao.createGroup(any[WorkbenchGroupName])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenReturn(Future.successful()) + + val ex = intercept[FireCloudException] { + Await.result(nihService.syncAllowlistAllUsers("TARGET"), Duration.Inf) + } + ex.getMessage should include(errorMessage) + } + + "syncAllNihWhitelistsAllUsers" should "sync all allowlists for all users" in { + mockEcmUsers() + when(thurloeDao.getAllUserValuesForKey(any[String])).thenReturn(Future.successful(Map.empty)) + + val targetEmailsToSync = Set(WorkbenchEmail(userTcgaAndTarget.email.value), WorkbenchEmail(userTargetOnly.email.value)) + val tcgaUsersToSync = Set(WorkbenchEmail(userTcgaAndTarget.email.value), WorkbenchEmail(userTcgaOnly.email.value)) + val nihStatus = Await.result(nihService.syncAllNihAllowlistsAllUsers(), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[StatusCode]].response + + nihStatus should be(StatusCodes.NoContent) + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "tcga-whitelist.txt") + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "target-whitelist.txt") + verify(samDao, times(1)).overwriteGroupMembers( + ArgumentMatchers.eq(WorkbenchGroupName("TARGET-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.argThat((list: List[WorkbenchEmail]) => list.toSet.equals(targetEmailsToSync)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, times(1)).overwriteGroupMembers( + ArgumentMatchers.eq(WorkbenchGroupName("TCGA-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.argThat((list: List[WorkbenchEmail]) => list.toSet.equals(tcgaUsersToSync)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + } + + "updateNihLinkAndSyncSelf" should "decode a JWT from Shibboleth and sync allowlists for a user" in { + mockShibbolethDAO() + mockEcmUsers() + mockThurloeUsers() + val user = userTcgaOnly + val userInfo = UserInfo(user.email.value, OAuth2BearerToken(user.id.value), Instant.now().plusSeconds(60).getEpochSecond, user.id.value) + val linkedAccount = userTcgaOnlyLinkedAccount + val jwt = jwtForUser(linkedAccount) + val (statusCode, nihStatus) = Await.result(nihService.updateNihLinkAndSyncSelf(userInfo, jwt), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[(StatusCode, NihStatus)]].response + + nihStatus.linkedNihUsername should be(Some(linkedAccount.linkedExternalId)) + nihStatus.linkExpireTime should be(Some(linkedAccount.linkExpireTime.getMillis / 1000L)) + nihStatus.datasetPermissions should be(Set( + NihDatasetPermission("BROKEN", authorized = false), + NihDatasetPermission("TARGET", authorized = false), + NihDatasetPermission("TCGA", authorized = true))) + + statusCode should be(StatusCodes.OK) + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "tcga-whitelist.txt") + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "target-whitelist.txt") + verify(samDao, times(1)).removeGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TARGET-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, times(1)).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TCGA-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, never()).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("this-doesnt-matter")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, never()).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("other-group")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + } + + it should "continue, but return an error of ECM returns an error" in { + mockShibbolethDAO() + mockThurloeUsers() + when(ecmDao.putLinkedEraAccount(any[LinkedEraAccount])(any[WithAccessToken])).thenReturn(Future.failed(new RuntimeException("ECM is down"))) + + val user = userTcgaOnly + val userInfo = UserInfo(user.email.value, OAuth2BearerToken(user.id.value), Instant.now().plusSeconds(60).getEpochSecond, user.id.value) + val linkedAccount = userTcgaOnlyLinkedAccount + val jwt = jwtForUser(linkedAccount) + val (statusCode, errorReport) = Await.result(nihService.updateNihLinkAndSyncSelf(userInfo, jwt), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[(StatusCode, ErrorReport)]].response + + errorReport.message should include("Error updating NIH link") + statusCode should be(StatusCodes.InternalServerError) + + verify(thurloeDao, times(1)).saveKeyValues(userInfo, NihLink(linkedAccount).propertyValueMap) + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "tcga-whitelist.txt") + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "target-whitelist.txt") + verify(samDao, times(1)).removeGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TARGET-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, times(1)).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TCGA-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, never()).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("this-doesnt-matter")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + } + + it should "continue, but return an error of Thurloe returns an error" in { + mockShibbolethDAO() + mockEcmUsers() + when(thurloeDao.saveKeyValues(any[UserInfo], any[Map[String, String]])).thenReturn(Future.successful(Failure(new RuntimeException("Thurloe is down")))) + + val user = userTcgaOnly + val userInfo = UserInfo(user.email.value, OAuth2BearerToken(user.id.value), Instant.now().plusSeconds(60).getEpochSecond, user.id.value) + val linkedAccount = userTcgaOnlyLinkedAccount + val jwt = jwtForUser(linkedAccount) + val (statusCode, errorReport) = Await.result(nihService.updateNihLinkAndSyncSelf(userInfo, jwt), Duration.Inf).asInstanceOf[PerRequest.RequestComplete[(StatusCode, ErrorReport)]].response + + errorReport.message should include("Error updating NIH link") + statusCode should be(StatusCodes.InternalServerError) + + // Tokens from Shibboleth are to the second, not millisecond + var expectedLinkedAccount = linkedAccount.copy(linkExpireTime = linkedAccount.linkExpireTime.minusMillis(linkedAccount.linkExpireTime.getMillisOfSecond)) + + verify(ecmDao, times(1)).putLinkedEraAccount(ArgumentMatchers.eq(expectedLinkedAccount))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "tcga-whitelist.txt") + verify(googleDao, times(1)).getBucketObjectAsInputStream(FireCloudConfig.Nih.whitelistBucket, "target-whitelist.txt") + verify(samDao, times(1)).removeGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TARGET-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, times(1)).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TCGA-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(samDao, never()).addGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("this-doesnt-matter")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + } + + "unlinkNihAccountAndSyncSelf" should "remove links from ECM and Thurloe, and sync allowlists" in { + mockEcmUsers() + mockThurloeUsers() + + val user = userTcgaOnly + val userInfo = UserInfo(user.email.value, OAuth2BearerToken(user.id.value), Instant.now().plusSeconds(60).getEpochSecond, user.id.value) + Await.result(nihService.unlinkNihAccountAndSyncSelf(userInfo), Duration.Inf) + + verify(samDao, times(1)).removeGroupMember( + ArgumentMatchers.eq(WorkbenchGroupName("TCGA-dbGaP-Authorized")), + ArgumentMatchers.eq(ManagedGroupRoles.Member), + ArgumentMatchers.eq(WorkbenchEmail(user.email.value)))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(ecmDao, times(1)).deleteLinkedEraAccount(ArgumentMatchers.eq(userInfo))(ArgumentMatchers.eq(UserInfo(adminAccessToken, ""))) + verify(thurloeDao, times(1)).deleteKeyValue(user.id.value, "linkedNihUsername", userInfo) + verify(thurloeDao, times(1)).deleteKeyValue(user.id.value, "linkExpireTime", userInfo) + + } + + private def mockSamUsers(): Unit = { + when(samDao.overwriteGroupMembers(any(), any(), any())(any())).thenReturn(Future.successful()) + when(samDao.listGroups(any[WithAccessToken])).thenAnswer(args => { + Future { + val userInfo = args.getArgument(0).asInstanceOf[WithAccessToken] + if (userInfo.accessToken.token.equals(adminAccessToken)) { + samGroupMemberships.keys.map(groupName => FireCloudManagedGroupMembership(groupName, groupName + "@firecloud.org", "member")).toList + } + val samUser = accessTokenToUser.get(userInfo.accessToken.token) + samUser.map(samUserToGroups(_).map(groupName => FireCloudManagedGroupMembership(groupName, groupName + "@firecloud.com", "member")).toList) + .getOrElse(List.empty) + } + }) + when(samDao.addGroupMember(any(), any(), any())(any())).thenReturn(Future.successful()) + when(samDao.removeGroupMember(any(), any(), any())(any())).thenReturn(Future.successful()) + when(samDao.isGroupMember(any[WorkbenchGroupName], any[UserInfo])).thenAnswer(args => Future { + val groupName = args.getArgument(0).asInstanceOf[WorkbenchGroupName] + val userInfo = args.getArgument(1).asInstanceOf[UserInfo] + samGroupMemberships.get(groupName.value).exists(_.exists(_.value == userInfo.id)) + }) + when(samDao.createGroup(any[WorkbenchGroupName])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenReturn(Future.successful()) + when(samDao.getUsersForIds(any[Seq[WorkbenchUserId]])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenAnswer(args => { + val userIds = args.getArgument(0).asInstanceOf[Seq[WorkbenchUserId]] + Future.successful(samUsers.filter(user => userIds.contains(WorkbenchUserId(user.id.value))).map(user => WorkbenchUserInfo(user.id.value, user.email.value))) + }) + + } + + private def mockEcmUsers(): Unit = { + when(ecmDao.getLinkedAccount(any[UserInfo])).thenAnswer(args => { + val userInfo = args.getArgument(0).asInstanceOf[UserInfo] + Future.successful(linkedAccountsBySamUserId.get(WorkbenchUserId(userInfo.id))) + }) + when(ecmDao.putLinkedEraAccount(any[LinkedEraAccount])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenReturn(Future.successful()) + when(ecmDao.deleteLinkedEraAccount(any[UserInfo])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenReturn(Future.successful()) + + when(ecmDao.getLinkedEraAccountForUsername(any[String])(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))).thenAnswer(args => { + val externalId = args.getArgument(0).asInstanceOf[String] + Future.successful(linkedAccountsByExternalId.get(externalId)) + }) + when(ecmDao.getActiveLinkedEraAccounts(ArgumentMatchers.eq(UserInfo(adminAccessToken, "")))) + .thenReturn(Future.successful(linkedAccountsBySamUserId.values.toSeq)) + } + + private def mockThurloeUsers(): Unit = { + when(thurloeDao.getAllKVPs(any[String], any[WithAccessToken])).thenAnswer(args => Future { + val userId = WorkbenchUserId(args.getArgument(0).asInstanceOf[String]) + val user = idToSamUser(userId) + val linkedEraAccount = linkedAccountsBySamUserId.get(userId) + Some(ProfileWrapper(userId.value, List( + FireCloudKeyValue(Some("contactEmail"), Some(user.email.value)), + FireCloudKeyValue(Some("linkedNihUsername"), linkedEraAccount.map(_.linkedExternalId)), + FireCloudKeyValue(Some("linkExpireTime"), linkedEraAccount.map(_.linkExpireTime.getMillis.toString)) + ))) + }) + when(thurloeDao.getAllUserValuesForKey(ArgumentMatchers.eq("email"))) + .thenReturn(Future.successful(samUsers.map(user => user.id.value -> user.email.value).toMap)) + when(thurloeDao.getAllUserValuesForKey(ArgumentMatchers.eq("linkedNihUsername"))).thenReturn(Future.successful(linkedAccountsBySamUserId.map(tup => (tup._1.value, tup._2.linkedExternalId)))) + when(thurloeDao.getAllUserValuesForKey(ArgumentMatchers.eq("linkExpireTime"))).thenReturn(Future.successful(linkedAccountsBySamUserId.map(tup => (tup._1.value, (tup._2.linkExpireTime.getMillis / 1000).toString)))) + when(thurloeDao.saveKeyValues(any[UserInfo], any[Map[String, String]])).thenReturn(Future.successful(Success())) + when(thurloeDao.saveKeyValues(any[String], any[WithAccessToken], any[Map[String, String]])).thenReturn(Future.successful(Success())) + when(thurloeDao.deleteKeyValue(any[String], any[String], any[WithAccessToken])).thenReturn(Future.successful(Success())) + } + + private def mockGoogleServicesDAO(): Unit = { + when(googleDao.getBucketObjectAsInputStream(ArgumentMatchers.eq(FireCloudConfig.Nih.whitelistBucket), any[String])).thenAnswer(args => { + val filename = args.getArgument(1).asInstanceOf[String] + val nihUsernames = filename match { + case "tcga-whitelist.txt" => Seq(userTcgaAndTargetLinkedAccount.linkedExternalId, userTcgaOnlyLinkedAccount.linkedExternalId) + case "target-whitelist.txt" => Seq(userTcgaAndTargetLinkedAccount.linkedExternalId, userTargetOnlyLinkedAccount.linkedExternalId) + case "broken-whitelist.txt" => Seq.empty + } + new ByteArrayInputStream(nihUsernames.mkString("\n").getBytes(StandardCharsets.UTF_8)) + }) + when(googleDao.getAdminUserAccessToken).thenReturn(adminAccessToken) + } + + private def mockShibbolethDAO(): Unit = { + when(shibbolethDao.getPublicKey()).thenReturn(Future.successful(pubKey)) + } + + val keypairGen = KeyPairGenerator.getInstance("RSA") + keypairGen.initialize(1024) + val keypair = keypairGen.generateKeyPair() + + val privKey: PrivateKey = keypair.getPrivate + val pubKey: String = s"-----BEGIN PUBLIC KEY-----\n${Base64.getEncoder.encodeToString(keypair.getPublic.getEncoded)}\n-----END PUBLIC KEY-----" + + private def jwtForUser(linkedEraAccount: LinkedEraAccount): JWTWrapper = { + val expiresInTheFuture: Long = linkedEraAccount.linkExpireTime.getMillis / 1000L + val issuedAt = Instant.ofEpochMilli(linkedEraAccount.linkExpireTime.minusDays(30).getMillis).getEpochSecond + val validStr = Jwt.encode( + JwtClaim(s"""{"eraCommonsUsername": "${linkedEraAccount.linkedExternalId}"}""").issuedAt(issuedAt).expiresAt(expiresInTheFuture), + privKey, + JwtAlgorithm.RS256) + JWTWrapper(validStr) + } + + private def genSamUser(): SamUser = { + SamUser( + WorkbenchUserId(Random.nextInt().toString), + Some(GoogleSubjectId(Random.nextInt().toString)), + WorkbenchEmail( UUID.randomUUID().toString + "@email.com"), + Some(AzureB2CId(UUID.randomUUID().toString)), + enabled = true, + Instant.now(), + Some(Instant.now()), + Instant.now()) + } + + + +} diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/service/UserServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/service/UserServiceSpec.scala index 8cb3d7416..f1a9f42d3 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/service/UserServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/service/UserServiceSpec.scala @@ -1,9 +1,8 @@ package org.broadinstitute.dsde.firecloud.service import akka.http.scaladsl.model.headers.OAuth2BearerToken -import akka.testkit.TestActorRef import org.broadinstitute.dsde.firecloud.{Application, FireCloudConfig} -import org.broadinstitute.dsde.firecloud.dataaccess.{MockCwdsDAO, MockResearchPurposeSupport, MockShareLogDAO} +import org.broadinstitute.dsde.firecloud.dataaccess.{DisabledExternalCredsDAO, MockCwdsDAO, MockResearchPurposeSupport, MockShareLogDAO} import org.broadinstitute.dsde.firecloud.mock.MockGoogleServicesDAO import org.broadinstitute.dsde.firecloud.model.{ProfileWrapper, UserInfo} import org.broadinstitute.dsde.firecloud.service.PerRequest.RequestComplete @@ -15,7 +14,7 @@ import scala.concurrent.duration._ class UserServiceSpec extends BaseServiceSpec with BeforeAndAfterEach { - val customApp = Application(agoraDao, new MockGoogleServicesFailedGroupsDAO(), ontologyDao, new MockRawlsDeleteWSDAO(), samDao, new MockSearchDeleteWSDAO(), new MockResearchPurposeSupport, thurloeDao, new MockShareLogDAO, shibbolethDao, new MockCwdsDAO) + val customApp = Application(agoraDao, new MockGoogleServicesFailedGroupsDAO(), ontologyDao, new MockRawlsDeleteWSDAO(), samDao, new MockSearchDeleteWSDAO(), new MockResearchPurposeSupport, thurloeDao, new MockShareLogDAO, shibbolethDao, new MockCwdsDAO, new DisabledExternalCredsDAO) val userServiceConstructor: (UserInfo) => UserService = UserService.constructor(customApp) diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/service/WorkspaceServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/service/WorkspaceServiceSpec.scala index fbfb7cedf..b84a2a373 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/service/WorkspaceServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/service/WorkspaceServiceSpec.scala @@ -1,7 +1,6 @@ package org.broadinstitute.dsde.firecloud.service import akka.http.scaladsl.model.{StatusCode, StatusCodes} import akka.http.scaladsl.model.headers.OAuth2BearerToken -import akka.testkit.TestActorRef import org.broadinstitute.dsde.firecloud.dataaccess._ import org.broadinstitute.dsde.firecloud.model.{AccessToken, WithAccessToken} import org.broadinstitute.dsde.firecloud.service.PerRequest.{RequestComplete, RequestCompleteWithHeaders} @@ -16,7 +15,7 @@ import scala.concurrent.{Await, ExecutionContext, Future} class WorkspaceServiceSpec extends BaseServiceSpec with BeforeAndAfterEach { - val customApp = Application(agoraDao, googleServicesDao, ontologyDao, new MockRawlsDeleteWSDAO(), samDao, new MockSearchDeleteWSDAO(), new MockResearchPurposeSupport, thurloeDao, new MockShareLogDAO, shibbolethDao, new MockCwdsDAO) + val customApp = Application(agoraDao, googleServicesDao, ontologyDao, new MockRawlsDeleteWSDAO(), samDao, new MockSearchDeleteWSDAO(), new MockResearchPurposeSupport, thurloeDao, new MockShareLogDAO, shibbolethDao, new MockCwdsDAO, new DisabledExternalCredsDAO) val workspaceServiceConstructor: (WithAccessToken) => WorkspaceService = WorkspaceService.constructor(customApp) diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/utils/EnabledUserDirectivesSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/utils/EnabledUserDirectivesSpec.scala index b7489a6f6..7408ef777 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/utils/EnabledUserDirectivesSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/utils/EnabledUserDirectivesSpec.scala @@ -158,7 +158,7 @@ class EnabledUserDirectivesSpec Get() ~> userEnabledRoute(samApiExceptionUser) ~> check { status shouldBe StatusCodes.ImATeapot val err = responseAs[ErrorReport] - err.message shouldBe s"Client Error (${StatusCodes.ImATeapot.intValue})" + err.message should include(s"Client Error (${StatusCodes.ImATeapot.intValue})") } } } diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/ApiServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/ApiServiceSpec.scala index a1a6289d9..44ec8cc25 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/ApiServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/ApiServiceSpec.scala @@ -36,11 +36,12 @@ trait ApiServiceSpec extends AnyFlatSpec with Matchers with ScalatestRouteTest w val shareLogDao: MockShareLogDAO val shibbolethDao: ShibbolethDAO val cwdsDao: CwdsDAO + val ecmDao: ExternalCredsDAO def actorRefFactory = system val nihServiceConstructor = NihService.constructor( - new Application(agoraDao, googleDao, ontologyDao, rawlsDao, samDao, searchDao, researchPurposeSupport, thurloeDao, shareLogDao, shibbolethDao, cwdsDao) + new Application(agoraDao, googleDao, ontologyDao, rawlsDao, samDao, searchDao, researchPurposeSupport, thurloeDao, shareLogDao, shibbolethDao, cwdsDao, ecmDao) ) _ } diff --git a/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiServiceSpec.scala b/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiServiceSpec.scala index b4ca0baf2..ae8be3e3b 100644 --- a/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiServiceSpec.scala +++ b/src/test/scala/org/broadinstitute/dsde/firecloud/webservice/NihApiServiceSpec.scala @@ -45,10 +45,10 @@ class NihApiServiceSpec extends ApiServiceSpec with BeforeAndAfterAll with SamMo //JWT for NIH username "not-on-whitelist" (don't ever add this to the mock whitelists in MockGoogleServicesDAO.scala) val validJwtNotOnWhitelist = JWTWrapper("eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJlcmFDb21tb25zVXNlcm5hbWUiOiJub3Qtb24td2hpdGVsaXN0IiwiaWF0IjoxNjE0ODc3NTc4MDB9.WpDrgtui5mOgDc5WvdWYC-l6vljGVyRI7DbBnpRYm7QOq00VLU6FI5YzVFe1eyjnHIqdz_KkkQD604Bi3G1qdyzhk_KKFCSeT4k5in-zS4Em_I2rcyUFs9DeHyFqVrBMZK8eZM_oKtSs23AtwGJASQ-sMvfXeXLcjTFuLWUdeiQEYedj9oOOA93ne-5Kaw9V7sR1foX-ybLDDHfHuAwTN2Vnvpmz0Qlk5osvvv-NunCo4M6A4fQ2FQWjrCwXk8-1N4Wf06dgDJ7ymsw9HtwHhzctVDzodaVlVU_RaC2gtSOWeD5nPaAJ7h6aNmNeLRmNwzCBm3TyPDY-qznPVM0DRg") - case class TestApiService(agoraDao: MockAgoraDAO, googleDao: MockGoogleServicesDAO, ontologyDao: MockOntologyDAO, rawlsDao: MockRawlsDAO, samDao: MockSamDAO, searchDao: MockSearchDAO, researchPurposeSupport: MockResearchPurposeSupport, thurloeDao: MockThurloeDAO, shareLogDao: MockShareLogDAO, shibbolethDao: MockShibbolethDAO, cwdsDao: CwdsDAO)(implicit val executionContext: ExecutionContext, implicit val materializer: Materializer) extends ApiServices + case class TestApiService(agoraDao: MockAgoraDAO, googleDao: MockGoogleServicesDAO, ontologyDao: MockOntologyDAO, rawlsDao: MockRawlsDAO, samDao: MockSamDAO, searchDao: MockSearchDAO, researchPurposeSupport: MockResearchPurposeSupport, thurloeDao: MockThurloeDAO, shareLogDao: MockShareLogDAO, shibbolethDao: MockShibbolethDAO, cwdsDao: CwdsDAO, ecmDao: ExternalCredsDAO)(implicit val executionContext: ExecutionContext, implicit val materializer: Materializer) extends ApiServices def withDefaultApiServices[T](testCode: TestApiService => T): T = { - val apiService = TestApiService(new MockAgoraDAO, new MockGoogleServicesDAO, new MockOntologyDAO, new MockRawlsDAO, new MockSamDAO, new MockSearchDAO, new MockResearchPurposeSupport, new MockThurloeDAO, new MockShareLogDAO, new MockShibbolethDAO, new MockCwdsDAO) + val apiService = TestApiService(new MockAgoraDAO, new MockGoogleServicesDAO, new MockOntologyDAO, new MockRawlsDAO, new MockSamDAO, new MockSearchDAO, new MockResearchPurposeSupport, new MockThurloeDAO, new MockShareLogDAO, new MockShibbolethDAO, new MockCwdsDAO, new DisabledExternalCredsDAO) testCode(apiService) }