From af4f8bb999c83c4855d4db20f3effab98519d966 Mon Sep 17 00:00:00 2001 From: Ari Brown Date: Fri, 1 Mar 2024 13:29:11 -0500 Subject: [PATCH] added sql loading code from metrics calculator to minerva --- minerva/__init__.py | 5 +++-- minerva/helpers.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/minerva/__init__.py b/minerva/__init__.py index de65a54..077dec0 100644 --- a/minerva/__init__.py +++ b/minerva/__init__.py @@ -1,4 +1,4 @@ -from .helpers import parallel_map, load_template +from .helpers import parallel_map, load_template, load_sql from .athena import Athena from .redshift import Redshift @@ -19,6 +19,7 @@ __all__ = [ "Pier", "Minerva", "parallel_map", - "load_template" + "load_template", + "load_sql" ] diff --git a/minerva/helpers.py b/minerva/helpers.py index 757cf50..05341df 100644 --- a/minerva/helpers.py +++ b/minerva/helpers.py @@ -41,3 +41,20 @@ def load_template(path, **params): return Template(query).render(**params) + +# `partition = {"field" => values}` +def partition_string(partition): + if not partition: + return "" + + parts = [f"{k} >= {min(v)} and {k} <= {max(v)}" for k, v in partition.items()] + string = " and ".join(parts) + + return string + + +# Wrapper to enable the inclusion of partitioning strings +def load_sql(path, **params): + params['partition_string'] = partition_string(params.get('partition', None)) + return m.load_template(path, **params) +