Skip to content

Commit

Permalink
Merge pull request #40 from kinghuang/drop-non-postgres
Browse files Browse the repository at this point in the history
Remove support for non-postgres adapters
  • Loading branch information
kudryk authored Jun 17, 2024
2 parents c481337 + 4c4463b commit b8cbaa3
Show file tree
Hide file tree
Showing 11 changed files with 73 additions and 918 deletions.
7 changes: 0 additions & 7 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,6 @@ Linear: closes FEA-XXX
Adapter to test:
<!-- Add relevant ones -->
- postgres
<!--
- snowflake
- bigquery
- redshift
- duckdb
- athena
-->

Python version to test:
<!-- Add relevant ones -->
Expand Down
27 changes: 0 additions & 27 deletions projects/adapter/integration_tests/profiles/redshift/profiles.yml

This file was deleted.

513 changes: 64 additions & 449 deletions projects/adapter/poetry.lock

Large diffs are not rendered by default.

5 changes: 0 additions & 5 deletions projects/adapter/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,6 @@ sqlalchemy = "^1.4.52"

# Adapters

## redshift
awswrangler = { version = ">=3.0.0", extras = ["redshift"], optional = true, python = ">=3.8" }
sqlalchemy-redshift = { version = "^0.8.9", optional = true }

# teleport
s3fs = { version = ">=2022.8.2", optional = true }

Expand All @@ -42,7 +38,6 @@ importlib-metadata = "^6.11.0"

[tool.poetry.extras]
postgres = []
redshift = ["awswrangler", "sqlalchemy-redshift"]
teleport = ["s3fs"]

[tool.poetry.group.dev]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
from typing import Any

import pandas as pd
import sqlalchemy
from contextlib import contextmanager
from dbt.adapters.base import BaseAdapter, BaseRelation, RelationType
from dbt.adapters.base.connections import AdapterResponse, Connection
Expand All @@ -21,30 +20,13 @@ def _get_alchemy_engine(adapter: BaseAdapter, connection: Connection) -> Any:
# The following code heavily depends on the implementation
# details of the known adapters, hence it can't work for
# arbitrary ones.
adapter_type = adapter.type()

sqlalchemy_kwargs = {}
format_url = lambda url: url

if adapter_type == "trino":
import dbt.adapters.fal_experimental.support.trino as support_trino

return support_trino.create_engine(adapter)

elif adapter_type == "sqlserver":
sqlalchemy_kwargs["creator"] = lambda *args, **kwargs: connection.handle
url = _SQLALCHEMY_DIALECTS.get(adapter_type, adapter_type) + "://"
url = format_url(url)
else:
message = (
f"dbt-fal does not support {adapter_type} adapter. ",
f"If you need {adapter_type} support, you can create an issue ",
"in our GitHub repository: https://github.com/fal-ai/fal. ",
"We will look into it ASAP.",
)
raise NotImplementedError(message)

return sqlalchemy.create_engine(url, **sqlalchemy_kwargs)
message = (
f"dbt-fal does not support {adapter_type} adapter. ",
f"If you need {adapter_type} support, you can create an issue ",
"in our GitHub repository: https://github.com/fal-ai/fal. ",
"We will look into it ASAP.",
)
raise NotImplementedError(message)


def drop_relation_if_it_exists(adapter: BaseAdapter, relation: BaseRelation) -> None:
Expand All @@ -68,35 +50,10 @@ def write_df_to_relation(

adapter_type = adapter.type()

if adapter_type == "snowflake":
import dbt.adapters.fal_experimental.support.snowflake as support_snowflake

return support_snowflake.write_df_to_relation(adapter, dataframe, relation)

elif adapter_type == "bigquery":
import dbt.adapters.fal_experimental.support.bigquery as support_bq

return support_bq.write_df_to_relation(adapter, dataframe, relation)

elif adapter_type == "duckdb":
import dbt.adapters.fal_experimental.support.duckdb as support_duckdb

return support_duckdb.write_df_to_relation(adapter, dataframe, relation)

elif adapter_type == "postgres":
if adapter_type == "postgres":
import dbt.adapters.fal_experimental.support.postgres as support_postgres

return support_postgres.write_df_to_relation(adapter, dataframe, relation)
elif adapter.type() == "athena":
import dbt.adapters.fal_experimental.support.athena as support_athena

return support_athena.write_df_to_relation(
adapter, dataframe, relation, if_exists
)
elif adapter_type == "redshift":
import dbt.adapters.fal_experimental.support.redshift as support_redshift

return support_redshift.write_df_to_relation(adapter, dataframe, relation)

else:
with new_connection(adapter, "fal:write_df_to_relation") as connection:
Expand Down Expand Up @@ -132,36 +89,11 @@ def read_relation_as_df(adapter: BaseAdapter, relation: BaseRelation) -> pd.Data

adapter_type = adapter.type()

if adapter_type == "snowflake":
import dbt.adapters.fal_experimental.support.snowflake as support_snowflake

return support_snowflake.read_relation_as_df(adapter, relation)

elif adapter_type == "bigquery":
import dbt.adapters.fal_experimental.support.bigquery as support_bq

return support_bq.read_relation_as_df(adapter, relation)

elif adapter_type == "duckdb":
import dbt.adapters.fal_experimental.support.duckdb as support_duckdb

return support_duckdb.read_relation_as_df(adapter, relation)

elif adapter_type == "postgres":
if adapter_type == "postgres":
import dbt.adapters.fal_experimental.support.postgres as support_postgres

return support_postgres.read_relation_as_df(adapter, relation)

elif adapter.type() == "athena":
import dbt.adapters.fal_experimental.support.athena as support_athena

return support_athena.read_relation_as_df(adapter, relation)

elif adapter.type() == "redshift":
import dbt.adapters.fal_experimental.support.redshift as support_redshift

return support_redshift.read_relation_as_df(adapter, relation)

else:
with new_connection(adapter, "fal:read_relation_as_df") as connection:
alchemy_engine = _get_alchemy_engine(adapter, connection)
Expand Down

This file was deleted.

This file was deleted.

This file was deleted.

Loading

0 comments on commit b8cbaa3

Please sign in to comment.