Add new check that dataset is not null

georgetown-cset · Aug 16, 2023 · e05e07c · e05e07c
1 parent 4f3dba7
commit e05e07c
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/linkage_dag.py b/linkage_dag.py
@@ -456,6 +456,11 @@
             sql=f"select count(0) = 0 from {staging_dataset}.references where id is null or ref_id is null",
             use_legacy_sql = False
         ),
+        BigQueryCheckOperator(
+            task_id="no_null_datasets",
+            sql=f"select count(0) = 0 from {staging_dataset}.sources where dataset is null",
+            use_legacy_sql=False
+        ),
     ])
 
     # We're done! Checks passed, so copy to production and post success to slack