diff --git a/minerva/__init__.py b/minerva/__init__.py index 5432993..de65a54 100644 --- a/minerva/__init__.py +++ b/minerva/__init__.py @@ -1,4 +1,4 @@ -from .helpers import parallel_map, local, load_template +from .helpers import parallel_map, load_template from .athena import Athena from .redshift import Redshift @@ -19,7 +19,6 @@ __all__ = [ "Pier", "Minerva", "parallel_map", - "local", "load_template" ] diff --git a/minerva/athena.py b/minerva/athena.py index 36e853f..e35b409 100644 --- a/minerva/athena.py +++ b/minerva/athena.py @@ -8,10 +8,13 @@ import pyarrow.dataset import pprint import datetime import dask.dataframe as dd -from minerva import parallel_map, local, load_template +from minerva import parallel_map, load_template pp = pprint.PrettyPrinter(indent=4) +def local(path): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), path) + class Athena: def __init__(self, handler, output=None): self.handler = handler diff --git a/minerva/helpers.py b/minerva/helpers.py index 6842993..757cf50 100644 --- a/minerva/helpers.py +++ b/minerva/helpers.py @@ -35,11 +35,6 @@ def parallel_map(func=None, data=None, cores=8): return [val for r in res for val in r] -# Get full path of fname -def local(fname): - return os.path.join(inspect.stack()[0][1], fname) - - def load_template(path, **params): with open(path, 'r') as f: query = f.read() diff --git a/minerva/redshift.py b/minerva/redshift.py index 3724511..2b8bc99 100644 --- a/minerva/redshift.py +++ b/minerva/redshift.py @@ -62,10 +62,13 @@ class Execute: Database=self.redshift.database, ClusterIdentifier=self.redshift.cluster) else: + params = {"WorkgroupName": self.redshift.workgroup} + if self.redshift.secret: + params['SecretArn'] = self.redshift.secret + resp = self.client.execute_statement(Sql=self.query(), Database=self.redshift.database, - SecretArn=self.redshift.secret, - WorkgroupName=self.redshift.workgroup) + **params) self.query_id = resp['Id'] return resp