loosened version requirements; added redshift history function; enabled Minerva to work using instance IAM role instead of explicit profile

minerva/minerva.py

@@ -6,6 +6,17 @@ class Minerva:
         self.session = boto3.session.Session(profile_name=profile)
         self.s3      = m.S3(self)
 
+class Minerva:
+    def __init__(self, profile=None):
+        kwargs = {}
+        if profile:
+            kwargs["profile_name"] =  profile
+        else:
+            kwargs["region_name"] = "us-east-1"
+
+        self.session = boto3.session.Session(**kwargs)
+        self.s3      = m.S3(self)
+
 
     def athena(self, *args, **kwargs):
         return m.Athena(self, *args, **kwargs)

minerva/redshift.py

@@ -36,6 +36,20 @@ class Redshift:
         e.run()
         return e
 
+    def history(self, limit=100, status='FINISHED', next_token=''):
+        resp = self.client.list_statements(MaxResults = limit,
+                                           Status     = status,
+                                           NextToken  = next_token)
+
+        res = []
+        for statement in resp['Statements']:
+            q = Query(self, statement['QueryString'])
+            q.info_cache = statement
+            q.update_values()
+            res.append(q)
+        return res
+
+
 
 class Execute:
     """
@@ -97,13 +111,18 @@ class Execute:
             time.sleep(5)
             stat = self.status()
 
+        self.update_values()
+
+    def update_values(self):
+        self.status_cache = self.info_cache['Status']
+
         self.runtime = self.info_cache['UpdatedAt'] - self.info_cache['CreatedAt']
 
         if self.redshift.rpus:
             # $0.36 / RPU-hour
             self.cost = 0.36 * self.redshift.rpus * self.runtime.seconds / 3600.0
 
-        return stat # finalized state
+        return self.status_cache
 
 
 class Query(Execute):

pyproject.toml

@@ -17,11 +17,11 @@ minerva-console = "minerva.console:main"
 
 [tool.poetry.dependencies]
 python = ">3.9, <3.12"
-boto3 = "^1.26.161"
+boto3 = "^1.34.0"
 pyarrow = "^14.0.1"
 joblib = "^1.1.0"
 fabric = "^3.0.0"
-s3fs = "2023.6.0"
+s3fs = ">2023.6.0"
 mako = ">1.2.0"
 dask = ">2023.11.0"
 distributed = ">2023.11.0"