fixing local paths and redshift credentials

minerva/__init__.py

@@ -1,4 +1,4 @@
-from .helpers import parallel_map, local, load_template
+from .helpers import parallel_map, load_template
 
 from .athena   import Athena
 from .redshift import Redshift
@@ -19,7 +19,6 @@ __all__ = [
     "Pier",
     "Minerva",
     "parallel_map",
-    "local",
     "load_template"
 ]
 

minerva/athena.py

@@ -8,10 +8,13 @@ import pyarrow.dataset
 import pprint
 import datetime
 import dask.dataframe as dd
-from minerva import parallel_map, local, load_template
+from minerva import parallel_map, load_template
 
 pp = pprint.PrettyPrinter(indent=4)
 
+def local(path):
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), path)
+
 class Athena:
     def __init__(self, handler, output=None):
         self.handler = handler

minerva/helpers.py

@@ -35,11 +35,6 @@ def parallel_map(func=None, data=None, cores=8):
     return [val for r in res for val in r]
 
 
-# Get full path of fname
-def local(fname):
-    return os.path.join(inspect.stack()[0][1], fname)
-
-
 def load_template(path, **params):
     with open(path, 'r') as f:
         query = f.read()

minerva/redshift.py

@@ -62,10 +62,13 @@ class Execute:
                                                   Database=self.redshift.database,
                                                   ClusterIdentifier=self.redshift.cluster)
         else:
+            params = {"WorkgroupName": self.redshift.workgroup}
+            if self.redshift.secret:
+                params['SecretArn'] = self.redshift.secret
+
             resp  = self.client.execute_statement(Sql=self.query(),
                                                   Database=self.redshift.database,
-                                                  SecretArn=self.redshift.secret,
+                                                  **params)
-                                                  WorkgroupName=self.redshift.workgroup)
 
         self.query_id = resp['Id']
         return resp