diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..db56a1a0c --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# This is a CODEOWNERS file for unitycatalog +# See https://help.github.com/articles/about-codeowners/ for more information + +docs/ @MrPowers +mkdocs.yml @MrPowers diff --git a/.github/workflows/release-workflow.yml b/.github/workflows/release-workflow.yml index dd4bdf7bd..fb3e87370 100644 --- a/.github/workflows/release-workflow.yml +++ b/.github/workflows/release-workflow.yml @@ -10,10 +10,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Build tarball diff --git a/.github/workflows/tarball-gen-tests.yml b/.github/workflows/tarball-gen-tests.yml index 201ca7777..8ecc39d93 100644 --- a/.github/workflows/tarball-gen-tests.yml +++ b/.github/workflows/tarball-gen-tests.yml @@ -12,10 +12,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Build tarball diff --git a/.github/workflows/tutorial-tests.yml b/.github/workflows/tutorial-tests.yml index fbaafb3bf..3010c2aaa 100644 --- a/.github/workflows/tutorial-tests.yml +++ b/.github/workflows/tutorial-tests.yml @@ -15,10 +15,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Set up Python diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 193157469..04639aa0d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -19,10 +19,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Run tests with coverage @@ -36,10 +36,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' - name: Generate OpenAPI Client and Server Model classes diff --git a/.gitignore b/.gitignore index d31aca2c5..1c86456b6 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ api/.openapi-generator build/sbt-launch-* venv server_pid.txt -site \ No newline at end of file +site +python_engine.log diff --git a/README.md b/README.md index 996dbbab4..8a6cbea35 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ Let's take Unity Catalog for spin. In this guide, we are going to do the followi ### Prerequisites You have to ensure that your local environment has the following: - Clone this repository. -- Ensure the `JAVA_HOME` environment variable your terminal is configured to point to JDK11+. +- Ensure the `JAVA_HOME` environment variable your terminal is configured to point to JDK17+. - Compile the project using `build/sbt package` ### Run the UC Server @@ -125,11 +125,11 @@ SELECT * from unity.default.numbers; You should see the tables listed and the contents of the `numbers` table printed. To quit DuckDB, run the command `Ctrl+D` or type `.exit` in the DuckDB shell. -## Full Tutorial +## CLI tutorial -You can read Delta Uniform tables from Spark via Iceberg REST Catalog APIs, +You can interact with a Unity Catalog server to create and manage catalogs, schemas and tables, operate on volumes and functions from the CLI, and much more. -See the full [tutorial](docs/tutorial.md) for more details. +See the [cli usage](docs/usage/cli.md) for more details. ## APIs and Compatibility - Open API specification: The Unity Catalog Rest API is documented [here](api). @@ -143,7 +143,7 @@ See the full [tutorial](docs/tutorial.md) for more details. This will create a tarball in the `target` directory. See the full [deployment guide](docs/deployment.md) for more details. ## Compiling and testing -- Install JDK 11 by whatever mechanism is appropriate for your system, and +- Install JDK 17 by whatever mechanism is appropriate for your system, and set that version to be the default Java version (e.g., by setting env variable JAVA_HOME) - To compile all the code without running tests, run the following: diff --git a/build.sbt b/build.sbt index 06e42caf7..21472d197 100644 --- a/build.sbt +++ b/build.sbt @@ -14,17 +14,17 @@ lazy val commonSettings = Seq( organization := orgName, // Compilation configs initialize := { - // Assert that the JVM is at least Java 11 + // Assert that the JVM is at least Java 17 val _ = initialize.value // ensure previous initializations are run assert( - sys.props("java.specification.version").toDouble >= 11, - "Java 11 or above is required to run this project.") + sys.props("java.specification.version").toDouble >= 17, + "Java 17 or above is required to run this project.") }, Compile / compile / javacOptions ++= Seq( "-Xlint:deprecation", "-Xlint:unchecked", - "-source", "1.8", - "-target", "1.8", + "-source", "17", + "-target", "17", "-g:source,lines,vars", ), resolvers += Resolver.mavenLocal, @@ -187,13 +187,30 @@ lazy val server = (project in file("server")) // Iceberg REST Catalog dependencies "org.apache.iceberg" % "iceberg-core" % "1.5.2", + "org.apache.iceberg" % "iceberg-aws" % "1.5.2", + "software.amazon.awssdk" % "s3" % "2.24.0", "io.vertx" % "vertx-core" % "4.3.5", "io.vertx" % "vertx-web" % "4.3.5", "io.vertx" % "vertx-web-client" % "4.3.5", // Test dependencies - "junit" % "junit" % "4.13.2" % Test, + "junit" % "junit" % "4.13.2" % Test, // TODO: update tests to junit5 and remove this + "org.junit.jupiter" % "junit-jupiter" % "5.10.1" % Test, + "org.mockito" % "mockito-core" % "4.11.0" % Test, + "org.mockito" % "mockito-inline" % "4.11.0" % Test, + "org.mockito" % "mockito-junit-jupiter" % "4.11.0" % Test, "com.github.sbt" % "junit-interface" % "0.13.3" % Test, + "com.adobe.testing" % "s3mock-junit5" % "2.11.0" % Test + exclude("ch.qos.logback", "logback-classic") + exclude("org.apache.logging.log4j", "log4j-to-slf4j") + // the following are runtime test dependencies we exclude here + // in order to not to set off the licences check, but then + // add back below as provided + exclude("jakarta.annotation", "jakarta.annotation-api") + exclude("jakarta.servlet", "jakarta.servlet-api") + exclude("jakarta.websocket", "jakarta.websocket-api"), + "jakarta.servlet" % "jakarta.servlet-api" % "4.0.4" % Provided, + "javax.xml.bind" % "jaxb-api" % "2.3.1" % Provided ), Compile / compile / javacOptions ++= Seq( diff --git a/docs/assets/images/uc-3-level.png b/docs/assets/images/uc-3-level.png new file mode 100644 index 000000000..3321151ec Binary files /dev/null and b/docs/assets/images/uc-3-level.png differ diff --git a/docs/assets/images/uc_java_version.png b/docs/assets/images/uc_java_version.png index ca0dc1ebc..4226803e2 100644 Binary files a/docs/assets/images/uc_java_version.png and b/docs/assets/images/uc_java_version.png differ diff --git a/docs/quickstart.md b/docs/quickstart.md index adfe92269..30a7bc53a 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -16,7 +16,7 @@ Change into the `unitycatalog` directory and run `bin/start-uc-server` to instan Well, that was pretty easy! -To run Unity Catalog, you need Java 11 installed on your machine. You can always run the `java --version` command to verify that you have the right version of Java installed. +To run Unity Catalog, you need Java 17 installed on your machine. You can always run the `java --version` command to verify that you have the right version of Java installed. ![UC Java Version](./assets/images/uc_java_version.png) @@ -42,6 +42,8 @@ Unity Catalog stores all assets in a 3-level namespace: 2. schema 3. assets like tables, volumes, functions, etc. +![UC 3 Level](./assets/images/uc-3-level.png) + Here's an example Unity Catalog instance: ![UC Example Catalog](./assets/images/uc_example_catalog.png) diff --git a/docs/usage/functions.md b/docs/usage/functions.md index 72dc36b08..f5783be46 100644 --- a/docs/usage/functions.md +++ b/docs/usage/functions.md @@ -1,56 +1,258 @@ # Unity Catalog Functions -You can register functions in Unity Catalog schemas. +This page shows you how to use Unity Catalog to store, access and govern Functions. -Persisting functions is good for reusing code and applying permissions or filters. +Functions are units of saved logic that return a scalar value or a set of rows. + +Using Unity Catalog to store your Functions is great for: + +1. reusing code, and +2. applying permissions or filters. The following diagram shows an example of a Unity Catalog instance with two functions: `sum` and `my_function`: ![UC Functions](../assets/images/uc_functions.png) -**Display functions** +Let's look at how this works. + +## Set Up + +We'll use a local Unity Catalog server to get started. The default local UC server comes with some sample data. + +> If this is your first time spinning up a UC server, you might want to check out the [Quickstart](../quickstart.md) first. + +Spin up a local UC server by running the following code in a terminal from the root directory of your local `unitycatalog` repository: + +```sh +bin/start-uc-server +``` + +Now open a second terminal window to start working with your Unity Catalog instance. + +## Inspecting Functions -Let's list the functions. +You can list the functions in your UC namespace using: ```sh bin/uc function list --catalog unity --schema default ``` -You should see a few functions. Let's get the metadata of one of these functions. +You should see something that looks like: + +``` +┌────────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┬────────┐ +│ NAME │CATALOG_│SCHEMA_N│INPUT_PA│DATA_TYP│FULL_DAT│RETURN_P│ROUTINE_│ROUTINE_│ROUTINE_│PARAMETE│IS_DETER│SQL_DATA│IS_NULL_│SECURITY│SPECIFIC│COMMENT │PROPERTI│FULL_NAM│CREATED_│UPDATED_│FUNCTION│EXTERNAL│ +│ │ NAME │ AME │ RAMS │ E │ A_TYPE │ ARAMS │ BODY │DEFINITI│DEPENDEN│R_STYLE │MINISTIC│_ACCESS │ CALL │ _TYPE │ _NAME │ │ ES │ E │ AT │ AT │ _ID │_LANGUAG│ +│ │ │ │ │ │ │ │ │ ON │ CIES │ │ │ │ │ │ │ │ │ │ │ │ │ E │ +├────────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┤ +│sum │unity │default │{"par...│INT │INT │null │EXTERNAL│t = x...│null │S │true │NO_SQL │false │DEFINER │sum │Adds ...│null │unity...│17183...│null │8e83e...│python │ +├────────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┼────────┤ +│lowercase │unity │default │{"par...│STRING │STRING │null │EXTERNAL│g = s...│null │S │true │NO_SQL │false │DEFINER │lower...│Conve...│null │unity...│17183...│null │33d81...│python │ +└────────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┴────────┘ +``` + +You can get the metadata of one of these functions using: ```sh bin/uc function get --full_name unity.default.sum ``` -![UC Function Metadata](../assets/images/uc_function_metadata.png) +You should see something that looks like: -In the printed metadata, pay attention to the columns `input_parameters`, `external_language`, and `routine_definition`. +``` +┌────────────────────┬────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ KEY │ VALUE │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│NAME │sum │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│CATALOG_NAME │unity │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SCHEMA_NAME │default │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│INPUT_PARAMS │{"parameters":[{"name":"x","type_text":"int","type_json":"{\"name\":\"x\",\"type\":\"integer\",\"nul│ +│ │lable\":false,\"metadata\":{}}","type_name":"INT","type_precision":null,"type_scale":null,"type_inte│ +│ │rval_type":null,"position":0,"parameter_mode":"IN","parameter_type":"PARAM","parameter_default":null│ +│ │,"comment":null},{"name":"y","type_text":"int","type_json":"{\"name\":\"y\",\"type\":\"integer\",\"n│ +│ │ullable\":false,\"metadata\":{}}","type_name":"INT","type_precision":null,"type_scale":null,"type_in│ +│ │terval_type":null,"position":1,"parameter_mode":"IN","parameter_type":"PARAM","parameter_default":nu│ +│ │ll,"comment":null},{"name":"z","type_text":"int","type_json":"{\"name\":\"z\",\"type\":\"integer\",\│ +│ │"nullable\":false,\"metadata\":{}}","type_name":"INT","type_precision":null,"type_scale":null,"type_│ +│ │interval_type":null,"position":2,"parameter_mode":"IN","parameter_type":"PARAM","parameter_default":│ +│ │null,"comment":null}]} │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│DATA_TYPE │INT │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│FULL_DATA_TYPE │INT │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│RETURN_PARAMS │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ROUTINE_BODY │EXTERNAL │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ROUTINE_DEFINITION │t = x + y + z\nreturn t │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ROUTINE_DEPENDENCIES│null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│PARAMETER_STYLE │S │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│IS_DETERMINISTIC │true │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SQL_DATA_ACCESS │NO_SQL │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│IS_NULL_CALL │false │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SECURITY_TYPE │DEFINER │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SPECIFIC_NAME │sum │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│COMMENT │Adds two numbers. │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│PROPERTIES │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│FULL_NAME │unity.default.sum │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│CREATED_AT │1718315581372 │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│UPDATED_AT │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│FUNCTION_ID │8e83e2d9-e523-46a1-b69c-8fe9212f1057 │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│EXTERNAL_LANGUAGE │python │ +└────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` -This seems like a simple python function that takes 3 arguments and returns the sum of them. Let's try calling this function. +The `routine definition` is probably most helpful here: `t = x + y + z\nreturn t ` -Behind the scenes, the invocation of the function is achieved by calling the python script at `etc/data/function/python_engine.py` with the function name and arguments. +It looks like this functions takes in 3 arguments and returns their sum. The `DATA_TYPE` field tells us that the inputs should be of `INT` data type. + +## Calling Functions from Unity Catalog + +Let's try calling this function. + +We'll use `function call` to reference the Function by its full name and pass 3 input parameters: ```sh bin/uc function call --full_name unity.default.sum --input_params "1,2,3" ``` -![UC Invoke Function](../assets/images/uc_invoke_function.png) +This should return: + +`6` + +Nicely done! You have called a Function stored in your Unity Catalog. -Voila! You have invoked a function stored in UC. +## Adding Functions to Unity Catalog -**Create function** +Let's create a new function and add it to your Unity Catalog. -Let's try and create a new function. +We'll start with a basic example using only scalar values. + +Suppose you want to register the following function to Unity Catalog: `c = a * b` + +To do so, define a new Function by its full name, specify the data type of the output, the input parameters and their data types, and ```sh bin/uc function create --full_name unity.default.my_function \ --data_type INT --input_params "a int, b int" --def "c=a*b\nreturn c" ``` -You can test out the newly created function by invoking it. +This should output something like: + +```sh + +┌────────────────────┬────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ KEY │ VALUE │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│NAME │myFunction │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│CATALOG_NAME │unity │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SCHEMA_NAME │default │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│INPUT_PARAMS │{"parameters":[{"name":"a","type_text":"int","type_json":"{\"name\":\"a\",\"type\":\"integer\",\"nul│ +│ │lable\":true,\"metadata\":{}}","type_name":"INT","type_precision":null,"type_scale":null,"type_inter│ +│ │val_type":null,"position":0,"parameter_mode":null,"parameter_type":null,"parameter_default":null,"co│ +│ │mment":null},{"name":"b","type_text":"int","type_json":"{\"name\":\"b\",\"type\":\"integer\",\"nulla│ +│ │ble\":true,\"metadata\":{}}","type_name":"INT","type_precision":null,"type_scale":null,"type_interva│ +│ │l_type":null,"position":1,"parameter_mode":null,"parameter_type":null,"parameter_default":null,"comm│ +│ │ent":null}]} │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│DATA_TYPE │INT │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│FULL_DATA_TYPE │INT │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│RETURN_PARAMS │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ROUTINE_BODY │EXTERNAL │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ROUTINE_DEFINITION │c=a*b\nreturn c │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│ROUTINE_DEPENDENCIES│null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│PARAMETER_STYLE │S │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│IS_DETERMINISTIC │true │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SQL_DATA_ACCESS │NO_SQL │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│IS_NULL_CALL │true │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SECURITY_TYPE │DEFINER │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│SPECIFIC_NAME │myFunction │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│COMMENT │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│PROPERTIES │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│FULL_NAME │unity.default.myFunction │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│CREATED_AT │1720516826170 │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│UPDATED_AT │null │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│FUNCTION_ID │012545ee-2a89-4534-b8e9-f41b09f4b2eb │ +├────────────────────┼────────────────────────────────────────────────────────────────────────────────────────────────────┤ +│EXTERNAL_LANGUAGE │python │ +└────────────────────┴────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +You can also store more complex functions. For example, you can import Python modules and use them in your function. + +Let's take the example below of a function that uses the Numpy library to simulate a random roll of dice: + +```python +def myPythonFunction(n_sides, n_dice): + import numpy as np + rolls=np.random.randint(1, n_sides + 1, size=n_dice) + return print(rolls)" +``` + +You can register this function to your Unity Catalog as follows: ```sh -bin/uc function call --full_name unity.default.my_function --input_params "2,9" +bin/uc function create --full_name unity.default.myPythonFunction --data_type INT --input_params "n_sides int, n_dice int" --language "python" --def "import numpy as np\nrolls=np.random.randint(1, n_sides + 1, size=n_dice)\nreturn print(rolls)" ``` -![UC Invoke Function 2](../assets/images/uc_invoke_function2.png) +And then call it with: + +```sh +bin/uc function call --full_name unity.default.myPythonFunction2 --input_params "6,1" +``` + +This will simulate rolling a single die with 6 sides. + +### Function Create Parameters + +`function create` takes the following parameters: + +Required Parameters: + +- `--full_name`: The full name of the table. The full name is the concatenation of the catalog name, schema name, and table/volume name separated by a dot. For example, catalog_name.schema_name.table_name. +- `--input_params`: The input parameters of the function, #ADD SYNTAX/FORMAT +- `--data_type`: The data type of the function, #ADD REF TO DTYPES + +Optional Parameters: + +- `--comment`: Comment/Description of the entity. +- `--def`: The routine definition of the function +- `--language`: The language of the function diff --git a/examples/cli/src/main/java/io/unitycatalog/cli/CatalogCli.java b/examples/cli/src/main/java/io/unitycatalog/cli/CatalogCli.java index 383e6d5e8..c034c6c26 100644 --- a/examples/cli/src/main/java/io/unitycatalog/cli/CatalogCli.java +++ b/examples/cli/src/main/java/io/unitycatalog/cli/CatalogCli.java @@ -14,7 +14,6 @@ import org.apache.commons.cli.CommandLine; import org.json.JSONObject; -import java.util.Collections; import java.util.List; import static io.unitycatalog.cli.utils.CliUtils.postProcessAndPrintOutput; diff --git a/examples/cli/src/main/java/io/unitycatalog/cli/FunctionCli.java b/examples/cli/src/main/java/io/unitycatalog/cli/FunctionCli.java index 5de3f91db..b01fcbbf2 100644 --- a/examples/cli/src/main/java/io/unitycatalog/cli/FunctionCli.java +++ b/examples/cli/src/main/java/io/unitycatalog/cli/FunctionCli.java @@ -25,7 +25,9 @@ public class FunctionCli { private static final ObjectMapper objectMapper = CliUtils.getObjectMapper(); private static ObjectWriter objectWriter; - public static void handle(CommandLine cmd, ApiClient apiClient) throws JsonProcessingException, ApiException { + public static void handle( + CommandLine cmd, + ApiClient apiClient) throws JsonProcessingException, ApiException { FunctionsApi functionsApi = new FunctionsApi(apiClient); String[] subArgs = cmd.getArgs(); objectWriter = CliUtils.getObjectWriter(cmd); diff --git a/examples/cli/src/main/java/io/unitycatalog/cli/utils/PythonInvoker.java b/examples/cli/src/main/java/io/unitycatalog/cli/utils/PythonInvoker.java index 633e57bb3..fcfa5a201 100644 --- a/examples/cli/src/main/java/io/unitycatalog/cli/utils/PythonInvoker.java +++ b/examples/cli/src/main/java/io/unitycatalog/cli/utils/PythonInvoker.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.stream.Collectors; public class PythonInvoker { @@ -27,17 +28,26 @@ public static String invokePython(FunctionInfo function, String scriptPath, Stri argsList.add(function.getRoutineDefinition()); // Retrieve and add parameters as arguments - List parameters = function.getInputParams() + List parameters = function + .getInputParams() .getParameters(); if (parameters == null || parameters.isEmpty()) { throw new ApiException("Function parameters not found."); } + if (args.length < parameters.size()) { + List names = parameters + .stream() + .skip(args.length) + .map(FunctionParameterInfo::getName) + .collect(Collectors.toList()); + throw new ApiException( + "Not enough parameters provided: " + args.length + ", expected: " + parameters.size() + ", missing: " + names); + } List paramNames = new ArrayList<>(); List argValues = new ArrayList<>(); - int index = 0; for (FunctionParameterInfo param : parameters) { paramNames.add(param.getName()); - String argument = args[index++]; + String argument = args[param.getPosition()]; if (param.getTypeName().equals(ColumnTypeName.INT)) { argValues.add(Integer.parseInt(argument)); } else if (param.getTypeName().equals(ColumnTypeName.DOUBLE)) { diff --git a/mkdocs.yml b/mkdocs.yml index 9afb793d5..f6b18604a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -3,7 +3,7 @@ edit_uri: edit/main/docs/ repo_name: unitycatalog/unitycatalog repo_url: https://github.com/unitycatalog/unitycatalog site_url: https://docs.unitycatalog.io -site_description: Open, Multi-modal Catalog for Data & AI +site_description: Open, Multi-modal Catalog for Data & AI site_author: unitycatalog nav: @@ -12,18 +12,18 @@ nav: - Usage: - CLI: usage/cli.md - Tables: - - UniForm: usage/tables/uniform.md + - UniForm: usage/tables/uniform.md - Volumes: usage/volumes.md - Functions: usage/functions.md - Server: usage/server.md - Integrations: - Daft: integrations/unity-catalog-daft.md - DuckDB: integrations/unity-catalog-duckdb.md - - Trino: integrations/unity-catalog-trino.md - Deployment: deployment.md plugins: - search + - material-plausible theme: name: material @@ -54,6 +54,23 @@ extra: social: - icon: fontawesome/brands/github link: https://github.com/unitycatalog/unitycatalog + analytics: + provider: plausible + domain: unitycatalog.io + feedback: + title: Was this page helpful? + ratings: + - icon: material/emoticon-happy-outline + name: This page was helpful + data: good + note: >- + Thanks for your feedback! + + - icon: material/emoticon-sad-outline + name: This page could be improved + data: bad + note: >- + Thanks for your feedback! markdown_extensions: - admonition diff --git a/project/build.properties b/project/build.properties index f716c53e0..88acb3059 100644 --- a/project/build.properties +++ b/project/build.properties @@ -33,4 +33,4 @@ # limitations under the License. # -sbt.version=1.10.0 +sbt.version=1.10.1 diff --git a/requirements-docs.txt b/requirements-docs.txt index 21f876b43..a1745e01c 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -1,2 +1,3 @@ mkdocs -mkdocs-material \ No newline at end of file +mkdocs-material +material-plausible-plugin \ No newline at end of file diff --git a/server/src/main/java/io/unitycatalog/server/UnityCatalogServer.java b/server/src/main/java/io/unitycatalog/server/UnityCatalogServer.java index ab67cc221..f0630bfe1 100644 --- a/server/src/main/java/io/unitycatalog/server/UnityCatalogServer.java +++ b/server/src/main/java/io/unitycatalog/server/UnityCatalogServer.java @@ -9,7 +9,16 @@ import com.linecorp.armeria.server.annotation.JacksonRequestConverterFunction; import com.linecorp.armeria.server.annotation.JacksonResponseConverterFunction; import com.linecorp.armeria.server.docs.DocService; -import io.unitycatalog.server.service.*; +import io.unitycatalog.server.service.CatalogService; +import io.unitycatalog.server.service.FunctionService; +import io.unitycatalog.server.service.IcebergRestCatalogService; +import io.unitycatalog.server.service.SchemaService; +import io.unitycatalog.server.service.TableService; +import io.unitycatalog.server.service.TemporaryTableCredentialsService; +import io.unitycatalog.server.service.TemporaryVolumeCredentialsService; +import io.unitycatalog.server.service.VolumeService; +import io.unitycatalog.server.service.iceberg.FileIOFactory; +import io.unitycatalog.server.service.iceberg.MetadataService; import io.unitycatalog.server.utils.RESTObjectMapper; import io.unitycatalog.server.utils.VersionUtils; import io.vertx.core.Verticle; @@ -73,9 +82,10 @@ private void addServices(ServerBuilder sb) { new JacksonRequestConverterFunction(icebergMapper); JacksonResponseConverterFunction icebergResponseConverter = new JacksonResponseConverterFunction(icebergMapper); + MetadataService metadataService = new MetadataService(new FileIOFactory()); sb.annotatedService( basePath + "iceberg", - new IcebergRestCatalogService(catalogService, schemaService, tableService), + new IcebergRestCatalogService(catalogService, schemaService, tableService, metadataService), icebergRequestConverter, icebergResponseConverter); } diff --git a/server/src/main/java/io/unitycatalog/server/persist/TableRepository.java b/server/src/main/java/io/unitycatalog/server/persist/TableRepository.java index 71cd14a69..c194bb1f5 100644 --- a/server/src/main/java/io/unitycatalog/server/persist/TableRepository.java +++ b/server/src/main/java/io/unitycatalog/server/persist/TableRepository.java @@ -336,7 +336,7 @@ public void deleteTable(Session session, UUID schemaId, String tableName) { try { FileUtils.deleteDirectory(tableInfoDAO.getUrl()); } catch (Throwable e) { - LOGGER.error("Error deleting table directory: " + tableInfoDAO.getUrl()); + LOGGER.error("Error deleting table directory: {}" ,tableInfoDAO.getUrl(), e); } } PropertyRepository.findProperties(session, tableInfoDAO.getId(), Constants.TABLE) diff --git a/server/src/main/java/io/unitycatalog/server/service/IcebergRestCatalogService.java b/server/src/main/java/io/unitycatalog/server/service/IcebergRestCatalogService.java index 27c271265..60eeddf36 100644 --- a/server/src/main/java/io/unitycatalog/server/service/IcebergRestCatalogService.java +++ b/server/src/main/java/io/unitycatalog/server/service/IcebergRestCatalogService.java @@ -4,46 +4,58 @@ import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.HttpResponse; import com.linecorp.armeria.common.HttpStatus; -import com.linecorp.armeria.server.annotation.*; +import com.linecorp.armeria.server.annotation.ExceptionHandler; +import com.linecorp.armeria.server.annotation.Get; +import com.linecorp.armeria.server.annotation.Head; +import com.linecorp.armeria.server.annotation.Param; +import com.linecorp.armeria.server.annotation.Post; +import com.linecorp.armeria.server.annotation.ProducesJson; import io.unitycatalog.server.exception.IcebergRestExceptionHandler; -import io.unitycatalog.server.model.*; +import io.unitycatalog.server.model.CatalogInfo; +import io.unitycatalog.server.model.ListCatalogsResponse; +import io.unitycatalog.server.model.ListSchemasResponse; import io.unitycatalog.server.model.ListTablesResponse; +import io.unitycatalog.server.model.SchemaInfo; import io.unitycatalog.server.persist.TableRepository; import io.unitycatalog.server.persist.utils.HibernateUtils; +import io.unitycatalog.server.service.iceberg.MetadataService; import io.unitycatalog.server.utils.JsonUtils; -import java.io.IOException; -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.stream.Collectors; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NoSuchViewException; import org.apache.iceberg.relocated.com.google.common.base.Splitter; -import org.apache.iceberg.rest.responses.*; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.GetNamespaceResponse; +import org.apache.iceberg.rest.responses.ListNamespacesResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.rest.responses.LoadViewResponse; import org.hibernate.Session; import org.hibernate.SessionFactory; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.stream.Collectors; + @ExceptionHandler(IcebergRestExceptionHandler.class) public class IcebergRestCatalogService { private final CatalogService catalogService; private final SchemaService schemaService; private final TableService tableService; + private final MetadataService metadataService; private final TableRepository tableRepository = TableRepository.getInstance(); private static final SessionFactory sessionFactory = HibernateUtils.getSessionFactory(); public IcebergRestCatalogService( - CatalogService catalogService, SchemaService schemaService, TableService tableService) { + CatalogService catalogService, SchemaService schemaService, TableService tableService, MetadataService metadataService) { this.catalogService = catalogService; this.schemaService = schemaService; this.tableService = tableService; + this.metadataService = metadataService; } // Config APIs @@ -175,8 +187,7 @@ public LoadTableResponse loadTable( throw new NoSuchTableException("Table does not exist: %s", namespace + "." + table); } - String metadataJson = new String(Files.readAllBytes(Paths.get(URI.create(metadataLocation)))); - TableMetadata tableMetadata = TableMetadataParser.fromJson(metadataLocation, metadataJson); + TableMetadata tableMetadata = metadataService.readTableMetadata(metadataLocation); return LoadTableResponse.builder().withTableMetadata(tableMetadata).build(); } diff --git a/server/src/main/java/io/unitycatalog/server/service/iceberg/FileIOFactory.java b/server/src/main/java/io/unitycatalog/server/service/iceberg/FileIOFactory.java new file mode 100644 index 000000000..38a239fe5 --- /dev/null +++ b/server/src/main/java/io/unitycatalog/server/service/iceberg/FileIOFactory.java @@ -0,0 +1,64 @@ +package io.unitycatalog.server.service.iceberg; + +import io.unitycatalog.server.exception.BaseException; +import io.unitycatalog.server.model.AwsCredentials; +import io.unitycatalog.server.persist.utils.ServerPropertiesUtils; +import io.unitycatalog.server.utils.TemporaryCredentialUtils; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.apache.iceberg.io.FileIO; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsSessionCredentials; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +import java.net.URI; +import java.util.Map; + +public class FileIOFactory { + + private static final String S3 = "s3"; + + public FileIOFactory() { + } + + // TODO: Cache fileIOs + public FileIO getFileIO(URI tableLocationUri) { + switch (tableLocationUri.getScheme()) { + case S3: return getS3FileIO(tableLocationUri); + // TODO: should we default/fallback to HadoopFileIO ? + default: return new SimpleLocalFileIO(); + } + } + + protected S3FileIO getS3FileIO(URI tableLocationUri) { + String region = ServerPropertiesUtils.getInstance().getProperty("aws.region", System.getenv("AWS_REGION")); + + // FIXME!! - proper credential vending and region settings + S3FileIO s3FileIO = new S3FileIO(() -> getS3Client(getAwsCredentialsProvider(tableLocationUri), region)); + + s3FileIO.initialize(Map.of()); + + return s3FileIO; + } + + protected S3Client getS3Client(AwsCredentialsProvider awsCredentialsProvider, String region) { + return S3Client.builder() + .region(Region.of(region)) + .credentialsProvider(awsCredentialsProvider) + .forcePathStyle(false) + .build(); + } + + private AwsCredentialsProvider getAwsCredentialsProvider(URI tableLocationUri) { + try { + AwsCredentials credentials = TemporaryCredentialUtils.findS3BucketConfig(tableLocationUri.toString()); + return StaticCredentialsProvider.create( + AwsSessionCredentials.create( + credentials.getAccessKeyId(),credentials.getSecretAccessKey(),credentials.getSessionToken())); + } catch (BaseException e) { + return DefaultCredentialsProvider.create(); + } + } +} diff --git a/server/src/main/java/io/unitycatalog/server/service/iceberg/MetadataService.java b/server/src/main/java/io/unitycatalog/server/service/iceberg/MetadataService.java new file mode 100644 index 000000000..c65b87eda --- /dev/null +++ b/server/src/main/java/io/unitycatalog/server/service/iceberg/MetadataService.java @@ -0,0 +1,25 @@ +package io.unitycatalog.server.service.iceberg; + +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.io.FileIO; + +import java.io.IOException; +import java.net.URI; + +public class MetadataService { + + private final FileIOFactory fileIOFactory; + + public MetadataService(FileIOFactory fileIOFactory) { + this.fileIOFactory = fileIOFactory; + } + + public TableMetadata readTableMetadata(String metadataLocation) { + URI metadataLocationUri = URI.create(metadataLocation); + // TODO: cache fileIO + FileIO fileIO = fileIOFactory.getFileIO(metadataLocationUri); + + return TableMetadataParser.read(fileIO, metadataLocation); + } +} diff --git a/server/src/main/java/io/unitycatalog/server/service/iceberg/SimpleLocalFileIO.java b/server/src/main/java/io/unitycatalog/server/service/iceberg/SimpleLocalFileIO.java new file mode 100644 index 000000000..815c9a75d --- /dev/null +++ b/server/src/main/java/io/unitycatalog/server/service/iceberg/SimpleLocalFileIO.java @@ -0,0 +1,23 @@ +package io.unitycatalog.server.service.iceberg; + +import org.apache.iceberg.Files; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.OutputFile; + +public class SimpleLocalFileIO implements FileIO { + @Override + public InputFile newInputFile(String path) { + return Files.localInput(path); + } + + @Override + public OutputFile newOutputFile(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteFile(String path) { + throw new UnsupportedOperationException(); + } +} diff --git a/server/src/main/java/io/unitycatalog/server/utils/ValidationUtils.java b/server/src/main/java/io/unitycatalog/server/utils/ValidationUtils.java index 25df92860..8791570a6 100644 --- a/server/src/main/java/io/unitycatalog/server/utils/ValidationUtils.java +++ b/server/src/main/java/io/unitycatalog/server/utils/ValidationUtils.java @@ -6,9 +6,9 @@ import java.util.regex.Pattern; public class ValidationUtils { - // Regex to reject names containing a period, space, forward-slash, C0 + DEL control characters - private static final Pattern INVALID_FORMAT = Pattern.compile("[\\.\\ \\/\\x00-\\x1F\\x7F]"); - private static final Integer MAX_NAME_LENGTH = 255; + // Regex to allow only alphanumeric characters, underscores, hyphens, and @ signs + private static final Pattern VALID_FORMAT = Pattern.compile("[a-zA-Z0-9_@-]+"); + private static final Integer MAX_NAME_LENGTH = 255; public static void validateSqlObjectName(String name) { if (name == null || name.isEmpty()) { @@ -19,7 +19,7 @@ public static void validateSqlObjectName(String name) { ErrorCode.INVALID_ARGUMENT, "Name cannot be longer than " + MAX_NAME_LENGTH + " characters"); } - if (INVALID_FORMAT.matcher(name).find()) { + if (!VALID_FORMAT.matcher(name).matches()) { throw new BaseException( ErrorCode.INVALID_ARGUMENT, "Name cannot contain a period, space, forward-slash, or control characters"); diff --git a/server/src/test/java/io/unitycatalog/server/iceberg/IcebergRestCatalogTest.java b/server/src/test/java/io/unitycatalog/server/service/IcebergRestCatalogTest.java similarity index 94% rename from server/src/test/java/io/unitycatalog/server/iceberg/IcebergRestCatalogTest.java rename to server/src/test/java/io/unitycatalog/server/service/IcebergRestCatalogTest.java index 9aae1e2cc..7ad2345b2 100644 --- a/server/src/test/java/io/unitycatalog/server/iceberg/IcebergRestCatalogTest.java +++ b/server/src/test/java/io/unitycatalog/server/service/IcebergRestCatalogTest.java @@ -1,12 +1,19 @@ -package io.unitycatalog.server.iceberg; - -import static org.assertj.core.api.Assertions.assertThat; +package io.unitycatalog.server.service; import com.linecorp.armeria.client.WebClient; import com.linecorp.armeria.common.AggregatedHttpResponse; import com.linecorp.armeria.common.auth.AuthToken; import io.unitycatalog.client.ApiException; -import io.unitycatalog.client.model.*; +import io.unitycatalog.client.model.CatalogInfo; +import io.unitycatalog.client.model.ColumnInfo; +import io.unitycatalog.client.model.ColumnTypeName; +import io.unitycatalog.client.model.CreateCatalog; +import io.unitycatalog.client.model.CreateSchema; +import io.unitycatalog.client.model.CreateTable; +import io.unitycatalog.client.model.DataSourceFormat; +import io.unitycatalog.client.model.SchemaInfo; +import io.unitycatalog.client.model.TableInfo; +import io.unitycatalog.client.model.TableType; import io.unitycatalog.server.base.BaseServerTest; import io.unitycatalog.server.base.catalog.CatalogOperations; import io.unitycatalog.server.base.schema.SchemaOperations; @@ -18,12 +25,6 @@ import io.unitycatalog.server.sdk.tables.SdkTableOperations; import io.unitycatalog.server.utils.RESTObjectMapper; import io.unitycatalog.server.utils.TestUtils; -import java.io.IOException; -import java.net.URISyntaxException; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.UUID; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; import org.apache.iceberg.exceptions.NoSuchTableException; @@ -38,6 +39,15 @@ import org.junit.Before; import org.junit.Test; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +import static org.assertj.core.api.Assertions.assertThat; + public class IcebergRestCatalogTest extends BaseServerTest { protected CatalogOperations catalogOperations; @@ -249,7 +259,7 @@ public void testTable() throws ApiException, IOException, URISyntaxException { assertThat(tableInfo.getTableId()).isNotNull(); session.load(tableInfoDAO, UUID.fromString(tableInfo.getTableId())); String metadataLocation = - Objects.requireNonNull(this.getClass().getResource("/metadata.json")).toURI().toString(); + Objects.requireNonNull(this.getClass().getResource("/iceberg.metadata.json")).toURI().toString(); tableInfoDAO.setUniformIcebergMetadataLocation(metadataLocation); session.merge(tableInfoDAO); tx.commit(); @@ -287,7 +297,7 @@ public void testTable() throws ApiException, IOException, URISyntaxException { LoadTableResponse loadTableResponse = RESTObjectMapper.mapper().readValue(resp.contentUtf8(), LoadTableResponse.class); assertThat(loadTableResponse.tableMetadata().metadataFileLocation()) - .isEqualTo(this.getClass().getResource("/metadata.json").toURI().toString()); + .isEqualTo(Objects.requireNonNull(this.getClass().getResource("/iceberg.metadata.json")).getPath()); } // List uniform tables diff --git a/server/src/test/java/io/unitycatalog/server/service/iceberg/MetadataServiceTest.java b/server/src/test/java/io/unitycatalog/server/service/iceberg/MetadataServiceTest.java new file mode 100644 index 000000000..35cf7d93c --- /dev/null +++ b/server/src/test/java/io/unitycatalog/server/service/iceberg/MetadataServiceTest.java @@ -0,0 +1,68 @@ +package io.unitycatalog.server.service.iceberg; + +import com.adobe.testing.s3mock.junit5.S3MockExtension; +import com.amazonaws.util.IOUtils; +import lombok.SneakyThrows; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; + +import java.util.Objects; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +@ExtendWith(S3MockExtension.class) +public class MetadataServiceTest { + @RegisterExtension + public static final S3MockExtension S3_MOCK = S3MockExtension.builder().silent().build(); + + public static final String TEST_BUCKET = "test-bucket"; + public static final String TEST_LOCATION = "test-bucket"; + public static final String TEST_SIMPLE_ICEBERG_V1_METADATA_FILE_NAME = "simple-v1-iceberg.metadata.json"; + + private final FileIOFactory mockFileIOFactory = mock(); + private final S3Client mockS3Client = S3_MOCK.createS3ClientV2(); + + private MetadataService metadataService; + + @SneakyThrows + @BeforeEach + public void setUp() { + metadataService = new MetadataService(mockFileIOFactory); + } + + @SneakyThrows + @Test + public void testGetTableMetadataFromS3() { + when(mockFileIOFactory.getFileIO(any())).thenReturn(new S3FileIO(() -> mockS3Client)); + mockS3Client.createBucket(builder -> builder.bucket(TEST_BUCKET).build()); + String simpleMetadataJson = IOUtils.toString( + Objects.requireNonNull(this.getClass().getResourceAsStream("/" + TEST_SIMPLE_ICEBERG_V1_METADATA_FILE_NAME))); + mockS3Client.putObject( + builder -> builder.bucket(TEST_BUCKET).key(TEST_LOCATION + "/" + TEST_SIMPLE_ICEBERG_V1_METADATA_FILE_NAME).build(), + RequestBody.fromString(simpleMetadataJson)); + + String metadataLocation = "s3://" + TEST_BUCKET + "/" + TEST_LOCATION + "/" + TEST_SIMPLE_ICEBERG_V1_METADATA_FILE_NAME; + TableMetadata tableMetadata = metadataService.readTableMetadata(metadataLocation); + assertThat(tableMetadata.uuid()).isEqualTo("11111111-2222-3333-4444-555555555555"); + } + + @SneakyThrows + @Test + public void testGetTableMetadataFromLocalFS() { + when(mockFileIOFactory.getFileIO(any())).thenReturn(new SimpleLocalFileIO()); + String metadataLocation = Objects.requireNonNull( + this.getClass().getResource("/iceberg.metadata.json")).toURI().toString(); + TableMetadata tableMetadata = metadataService.readTableMetadata(metadataLocation); + assertThat(tableMetadata.uuid()).isEqualTo("55d4dc69-5b14-4483-bfc8-f33b80f99f99"); + } + +} diff --git a/server/src/test/resources/metadata.json b/server/src/test/resources/iceberg.metadata.json similarity index 100% rename from server/src/test/resources/metadata.json rename to server/src/test/resources/iceberg.metadata.json diff --git a/server/src/test/resources/simple-v1-iceberg.metadata.json b/server/src/test/resources/simple-v1-iceberg.metadata.json new file mode 100644 index 000000000..9a4afc100 --- /dev/null +++ b/server/src/test/resources/simple-v1-iceberg.metadata.json @@ -0,0 +1,42 @@ +{ + "format-version": 1, + "table-uuid": "11111111-2222-3333-4444-555555555555", + "location": "s3://test-bucket/testLocation", + "last-updated-ms": 1720620264000, + "last-column-id": 3, + "schema": { + "type": "struct", + "fields": [ + { + "id": 1, + "name": "id", + "required": true, + "type": "string" + }, + { + "id": 2, + "name": "cat", + "required": true, + "type": "string", + "doc": "comment" + }, + { + "id": 3, + "name": "z", + "required": true, + "type": "long" + } + ] + }, + "partition-spec": [ + { + "name": "id", + "transform": "identity", + "source-id": 1, + "field-id": 1000 + } + ], + "properties": {}, + "current-snapshot-id": -1, + "snapshots": [] +} \ No newline at end of file