forked from svt/encore
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: support s3 urls for input and output
This commit add support for using s3 urls on the format s3://<BUCKET>/<KEY> in both input and output. If ans s3 URL is used as input, a presigned URL is created and used as input to ffmpeg. The duration of the presigned URLs can be controlled with the 'remote-files.s3.presignDurationSeconds' config property. If an s3 URL is used for 'outputFolder', output will first be stored locally and then uploaded to s3 once transcoding is finished. Aws credentials are read with DefaultCredentialsProvider, meaning aws credentials can be provided in a number of ways, see https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/auth/credentials/DefaultCredentialsProvider.html; Not that when using s3 urls for input, the presigned URLs will be shown in the logs. If this is not desirable, setting logging.config (or env variable LOGGING_CONFIG) to 'classpath:logback-json-mask-s3-presign.xml' will use a log config that masks the presign query parameters. Signed-off-by: Gustav Grusell <[email protected]>
- Loading branch information
Showing
19 changed files
with
476 additions
and
53 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
encore-common/src/main/kotlin/se/svt/oss/encore/S3RemoteFilesConfiguration.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package se.svt.oss.encore | ||
|
|
||
| import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty | ||
| import org.springframework.boot.context.properties.EnableConfigurationProperties | ||
| import org.springframework.context.annotation.Bean | ||
| import org.springframework.context.annotation.Configuration | ||
| import se.svt.oss.encore.service.remotefiles.s3.S3Properties | ||
| import se.svt.oss.encore.service.remotefiles.s3.S3RemoteFileHandler | ||
| import software.amazon.awssdk.regions.Region | ||
| import software.amazon.awssdk.services.s3.S3AsyncClient | ||
| import software.amazon.awssdk.services.s3.S3Configuration | ||
| import software.amazon.awssdk.services.s3.presigner.S3Presigner | ||
| import java.net.URI | ||
|
|
||
| @ConditionalOnProperty("remote-files.s3.enabled", havingValue = "true") | ||
| @EnableConfigurationProperties(S3Properties::class) | ||
| @Configuration | ||
| class S3RemoteFilesConfiguration { | ||
|
|
||
| @Bean | ||
| fun s3Region() = | ||
| Region.of(System.getProperty("aws.region") ?: System.getenv("AWS_REGION") ?: "us-east-1") | ||
|
|
||
| @Bean | ||
| fun s3Client(s3Region: Region, s3Properties: S3Properties) = S3AsyncClient.builder() | ||
| .region(s3Region) | ||
| .crossRegionAccessEnabled(true) | ||
| .multipartEnabled(true) | ||
| .serviceConfiguration( | ||
| S3Configuration.builder() | ||
| .pathStyleAccessEnabled(true) | ||
| .build() | ||
| ) | ||
| .apply { | ||
| if (!s3Properties.endpoint.isNullOrBlank()) { | ||
| endpointOverride(URI.create(s3Properties.endpoint)) | ||
| } | ||
| } | ||
| .build() | ||
|
|
||
| @Bean | ||
| fun s3Presigner(s3Region: Region, s3Properties: S3Properties) = S3Presigner.builder() | ||
| .region(s3Region) | ||
| .serviceConfiguration( | ||
| S3Configuration.builder() | ||
| .pathStyleAccessEnabled(true) | ||
| .build() | ||
| ) | ||
| .apply { | ||
| if (!s3Properties.endpoint.isNullOrBlank()) { | ||
| endpointOverride(URI.create(s3Properties.endpoint)) | ||
| } | ||
| } | ||
| .build() | ||
|
|
||
| @Bean | ||
| fun s3RemoteFileHandler(s3Client: S3AsyncClient, s3Presigner: S3Presigner, s3Properties: S3Properties) = | ||
| S3RemoteFileHandler(s3Client, s3Presigner, s3Properties) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
7 changes: 7 additions & 0 deletions
7
encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/RemoteFileHandler.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| package se.svt.oss.encore.service.remotefiles | ||
|
|
||
| interface RemoteFileHandler { | ||
| fun getAccessUri(uri: String): String | ||
| fun upload(localFile: String, remoteFile: String) | ||
| val protocols: List<String> | ||
| } |
55 changes: 55 additions & 0 deletions
55
encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/RemoteFileService.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| package se.svt.oss.encore.service.remotefiles | ||
|
|
||
| import mu.KotlinLogging | ||
| import org.springframework.stereotype.Service | ||
| import java.net.URI | ||
|
|
||
| @Service | ||
| class RemoteFileService(private val remoteFileHandlers: List<RemoteFileHandler>) { | ||
|
|
||
| private val log = KotlinLogging.logger {} | ||
|
|
||
| private val defaultHandler = DefaultHandler() | ||
|
|
||
| fun isRemoteFile(uriOrPath: String): Boolean { | ||
| val uri = URI.create(uriOrPath) | ||
| return !(uri.scheme.isNullOrEmpty() || uri.scheme.lowercase() == "file") | ||
| } | ||
|
|
||
| fun getAccessUri(uriOrPath: String): String { | ||
| val uri = URI.create(uriOrPath) | ||
| return getHandler(uri).getAccessUri(uriOrPath) | ||
| } | ||
|
|
||
| fun upload(localFile: String, remoteFile: String) { | ||
| val uri = URI.create(remoteFile) | ||
| getHandler(uri).upload(localFile, remoteFile) | ||
| } | ||
|
|
||
| private fun getHandler(uri: URI): RemoteFileHandler { | ||
| log.info { "Getting handler for uri $uri. Available protocols: ${remoteFileHandlers.flatMap {it.protocols} }" } | ||
| if (uri.scheme.isNullOrEmpty() || uri.scheme.lowercase() == "file") { | ||
| return defaultHandler | ||
| } | ||
| val handler = remoteFileHandlers.firstOrNull { it.protocols.contains(uri.scheme) } | ||
| if (handler != null) { | ||
| return handler | ||
| } | ||
| log.info { "No remote file handler found for protocol ${uri.scheme}. Using default handler." } | ||
| return defaultHandler | ||
| } | ||
|
|
||
| /** Handler user for protocols where no specific handler is defined. Works for local files and | ||
| * any protocols that ffmpeg supports natively */ | ||
| private class DefaultHandler : RemoteFileHandler { | ||
| override fun getAccessUri(uri: String): String { | ||
| return uri | ||
| } | ||
|
|
||
| override fun upload(localFile: String, remoteFile: String) { | ||
| // Do nothing | ||
| } | ||
|
|
||
| override val protocols: List<String> = emptyList() | ||
| } | ||
| } |
12 changes: 12 additions & 0 deletions
12
encore-common/src/main/kotlin/se/svt/oss/encore/service/remotefiles/s3/S3Properties.kt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| package se.svt.oss.encore.service.remotefiles.s3 | ||
|
|
||
| import org.springframework.boot.context.properties.ConfigurationProperties | ||
| import java.time.Duration | ||
|
|
||
| @ConfigurationProperties("remote-files.s3") | ||
| data class S3Properties( | ||
| val enabled: Boolean = false, | ||
| val endpoint: String = "", | ||
| val presignDurationSeconds: Long = Duration.ofHours(12).seconds, | ||
| val uploadTimeoutSeconds: Long = Duration.ofHours(1).seconds | ||
| ) |
Oops, something went wrong.