Merge pull request #113 from IndexSeek/linting-maintenance

lint: update ruff and pre-commit config

.gitignore

@@ -17,6 +17,7 @@ __pycache__/
 # PyVenv
 .env
 .venv
+venv
 
 # Linters cache
 .mypy_cache

.pre-commit-config.yml → .pre-commit-config.yaml

@@ -1,7 +1,7 @@
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: 'v0.0.265'
+    rev: 'v0.0.286'
     hooks:
       - id: ruff
   - repo: local

pyproject.toml

@@ -33,7 +33,7 @@ mkdocs = "^1.4.2"
 # Allow lines to be as long as 150 characters.
 line-length = 150
 ignore = ["D100"]
-required-version = "0.0.265"
+required-version = "0.0.286"
 
 [build-system]
 requires = ["poetry>=0.12"]

quinn/__init__.py

@@ -1,8 +1,5 @@
 """"""
 from .dataframe_helpers import *
 from .dataframe_validator import *
-from .schema_helpers import print_schema_as_code
 from .functions import *
-from .scala_to_pyspark import ScalaToPyspark
 from .transformations import *
-from .append_if_schema_identical import append_if_schema_identical

quinn/extensions/column_ext.py

@@ -36,7 +36,7 @@ def isFalse(self: Column) -> Column:
     :return: Column
     :rtype: Column
     """
-    return self == False
+    return self is False
 
 
 def isTrue(self: Column) -> Column:
@@ -51,7 +51,7 @@ def isTrue(self: Column) -> Column:
     :returns: Column object
     :rtype: Column
     """
-    return self == True
+    return self is True
 
 
 def isNullOrBlank(self: Column) -> Column:

quinn/functions.py

@@ -189,20 +189,20 @@ def _raise_if_invalid_day(day: str) -> None:
         raise ValueError(message)
 
 
-def approx_equal(col1: Column, col2: Column, threshhold: Number) -> Column:
+def approx_equal(col1: Column, col2: Column, threshold: Number) -> Column:
     """Compares two ``Column`` objects by checking if the difference between them
-    is less than a specified ``threshhold``.
+    is less than a specified ``threshold``.
 
     :param col1: the first ``Column``
     :type col1: Column
     :param col2: the second ``Column``
     :type col2: Column
-    :param threshhold: value to compare with
-    :type threshhold: Number
+    :param threshold: value to compare with
+    :type threshold: Number
     :return: Boolean ``Column`` with ``True`` indicating that ``abs(col1 -
-    col2)`` is less than ``threshhold``
+    col2)`` is less than ``threshold``
     """
-    return F.abs(col1 - col2) < threshhold
+    return F.abs(col1 - col2) < threshold
 
 
 def array_choice(col: Column) -> Column:
@@ -227,7 +227,7 @@ def regexp_extract_all(s: str, regexp: str) -> Optional[List[re.Match]]:
     :param regexp: string `re` pattern
     :return: List of matches
     """
-    return None if s == None else re.findall(regexp, s)
+    return None if s is None else re.findall(regexp, s)
 
 
 def uuid5(col: Column, namespace: uuid.UUID = uuid.NAMESPACE_DNS, extra_string: str = "") -> Column:

tests/extensions/test_dataframe_ext.py

@@ -1,8 +1,6 @@
 from functools import partial
 
 import chispa
-from chispa.schema_comparer import assert_basic_schema_equality
-from pyspark.sql.dataframe import DataFrame
 from pyspark.sql.functions import col
 
 from tests.conftest import auto_inject_fixtures

tests/test_append_if_schema_identical.py

@@ -5,7 +5,7 @@
 
 @auto_inject_fixtures("spark")
 def test_append_if_schema_identical(spark):
-    source_data = [(1, "capetown", "Alice"), (2, "delhi", "Bob")]
+    source_data = [(1, "cape town", "Alice"), (2, "delhi", "Bob")]
     target_data = [(3, "Charlie", "New York"), (4, "Dave", "Los Angeles")]
 
     source_df = spark.createDataFrame(source_data, schema=StructType([
@@ -21,4 +21,4 @@ def test_append_if_schema_identical(spark):
     ]))
 
     # Call the append_if_schema_identical function
-    appended_df = quinn.append_if_schema_identical(source_df, target_df)
+    quinn.append_if_schema_identical(source_df, target_df)

tests/test_dataframe_helpers.py

@@ -1,4 +1,3 @@
-from pyspark.sql.types import IntegerType, StringType
 
 import quinn
 from tests.conftest import auto_inject_fixtures

tests/test_functions.py

@@ -1,6 +1,5 @@
 import pytest
 
-import re
 
 import pyspark.sql.functions as F
 from pyspark.sql.types import *
@@ -305,7 +304,7 @@ def test_array_choice(spark):
         [(["a", "b", "c"],), (["a", "b", "c", "d"],), (["x"],), ([None],)],
         [("letters", ArrayType(StringType(), True), True)],
     )
-    actual_df = df.withColumn("random_letter", quinn.array_choice(F.col("letters")))
+    df.withColumn("random_letter", quinn.array_choice(F.col("letters")))
     # actual_df.show()
     # chispa.assert_column_equality(actual_df, "are_nums_approx_equal", "expected")
 

tests/test_transformations.py

@@ -5,8 +5,6 @@
 from tests.conftest import auto_inject_fixtures
 import chispa
 
-from functools import reduce
-
 
 @auto_inject_fixtures("spark")
 def describe_with_columns_renamed():