diff --git a/minerva/minerva.py b/minerva/minerva.py index c0c7028..d0e9e33 100644 --- a/minerva/minerva.py +++ b/minerva/minerva.py @@ -6,6 +6,17 @@ class Minerva: self.session = boto3.session.Session(profile_name=profile) self.s3 = m.S3(self) +class Minerva: + def __init__(self, profile=None): + kwargs = {} + if profile: + kwargs["profile_name"] = profile + else: + kwargs["region_name"] = "us-east-1" + + self.session = boto3.session.Session(**kwargs) + self.s3 = m.S3(self) + def athena(self, *args, **kwargs): return m.Athena(self, *args, **kwargs) diff --git a/minerva/redshift.py b/minerva/redshift.py index 4217cba..58eb4b2 100644 --- a/minerva/redshift.py +++ b/minerva/redshift.py @@ -36,6 +36,20 @@ class Redshift: e.run() return e + def history(self, limit=100, status='FINISHED', next_token=''): + resp = self.client.list_statements(MaxResults = limit, + Status = status, + NextToken = next_token) + + res = [] + for statement in resp['Statements']: + q = Query(self, statement['QueryString']) + q.info_cache = statement + q.update_values() + res.append(q) + return res + + class Execute: """ @@ -97,13 +111,18 @@ class Execute: time.sleep(5) stat = self.status() + self.update_values() + + def update_values(self): + self.status_cache = self.info_cache['Status'] + self.runtime = self.info_cache['UpdatedAt'] - self.info_cache['CreatedAt'] if self.redshift.rpus: # $0.36 / RPU-hour self.cost = 0.36 * self.redshift.rpus * self.runtime.seconds / 3600.0 - return stat # finalized state + return self.status_cache class Query(Execute): diff --git a/pyproject.toml b/pyproject.toml index 3e8b404..b9820ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,11 +17,11 @@ minerva-console = "minerva.console:main" [tool.poetry.dependencies] python = ">3.9, <3.12" -boto3 = "^1.26.161" +boto3 = "^1.34.0" pyarrow = "^14.0.1" joblib = "^1.1.0" fabric = "^3.0.0" -s3fs = "2023.6.0" +s3fs = ">2023.6.0" mako = ">1.2.0" dask = ">2023.11.0" distributed = ">2023.11.0"