From 5e604f07f737e2ce0675e497388b43d75f9dbb6c Mon Sep 17 00:00:00 2001
From: Ari Brown <ari@airintech.com>
Date: Tue, 24 Sep 2024 10:48:47 -0400
Subject: [PATCH] fixed repartition

---
 .gitlab-ci.yml          | 2 +-
 examples/repartition.py | 7 ++-----
 minerva/minerva.py      | 5 ++++-
 pyproject.toml          | 4 +++-
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1daeeb0..84d8a87 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -8,4 +8,4 @@ run:
     - poetry config http-basic.gitlab gitlab-ci-token "$CI_JOB_TOKEN"
     - poetry publish --repository gitlab
   tags:
-    - dind
+    - autoscale
diff --git a/examples/repartition.py b/examples/repartition.py
index 220f138..3751d13 100644
--- a/examples/repartition.py
+++ b/examples/repartition.py
@@ -38,7 +38,7 @@ def repartition(mach, agents):
     # Prep the info for the docker container
     variables = {"min_agent":   min(agents),
                  "max_agent":   max(agents),
-                 "source":      src_top_level,
+                 "source":      sorted_top,
                  "destination": dst_top_level,
                  "secondary_destination": None}
 
@@ -54,7 +54,7 @@ def repartition(mach, agents):
 #####################################
 # Prep the work
 # Find out how many hours there are in the dataset
-pool_size = 5
+pool_size = 1
 
 objs  = s.m.s3.ls(src_top_level + "year=")
 hours = set(["s3://" + '/'.join([o.bucket_name, *o.key.split("/")[0:-1]])
@@ -87,9 +87,6 @@ try:
     # Second part: repartition
     pool.run(repartition, data=groups)
 
-    import IPython
-    IPython.embed()
-
 finally:
     pool.terminate()
 
diff --git a/minerva/minerva.py b/minerva/minerva.py
index 8e1203c..0cd7252 100644
--- a/minerva/minerva.py
+++ b/minerva/minerva.py
@@ -2,13 +2,16 @@ import boto3
 import minerva as m
 
 class Minerva:
-    def __init__(self, profile=None):
+    def __init__(self, profile=None, region=None):
         kwargs = {}
         if profile:
             kwargs["profile_name"] =  profile
         else:
             kwargs["region_name"] = "us-east-1"
 
+        if region:
+            kwargs["region_name"] = region
+
         self.session = boto3.session.Session(**kwargs)
         self.s3      = m.S3(self)
 
diff --git a/pyproject.toml b/pyproject.toml
index 60540a4..08226e4 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,7 +16,7 @@ readme = "README.md"
 minerva-console = "minerva.console:main"
 
 [tool.poetry.dependencies]
-python = ">3.9, <3.12"
+python = ">3.9"
 boto3 = "^1.34.0"
 pyarrow = "^14.0.1"
 joblib = "^1.1.0"
@@ -25,3 +25,5 @@ s3fs = ">2023.6.0"
 mako = ">1.2.0"
 dask = ">2023.11.0"
 distributed = ">2023.11.0"
+pandas = ">2.0.0"
+numpy = ">1.26.0"