beginning to add web visualization of pools

examples/repartition.py

@@ -47,7 +47,7 @@ def repartition(mach, agents):
 #####################################
 # Prep the work
 # Find out how many hours there are in the dataset
-pool_size = 1
+pool_size = 5
 
 objs  = s.m.s3.ls(src_top_level + "year=")
 hours = set(["s3://" + '/'.join([o.bucket_name, *o.key.split("/")[0:-1]])
@@ -66,7 +66,7 @@ try:
     #######################################
     # Create the machines
     # This also waits for them to be made
-    pool      = m.Pool(s.worker, pool_size)
+    pool      = m.Pool(s.worker, pool_size, web=True)
 
     ########################################
     # Now that we have the pool, put them to work
@@ -74,10 +74,13 @@ try:
     # doing that until the list is empty
 
     # First part: sort the individual files
-    pool.run(sort_hour, data=hours)
+    #pool.run(sort_hour, data=hours)
 
     # Second part: repartition
-    pool.run(repartition, data=groups)
+    #pool.run(repartition, data=groups)
+
+    import IPython
+    IPython.embed()
 
 finally:
     pool.terminate()

minerva/minerva.py

@@ -1,11 +1,6 @@
 import boto3
 import minerva as m
 
-class Minerva:
-    def __init__(self, profile):
-        self.session = boto3.session.Session(profile_name=profile)
-        self.s3      = m.S3(self)
-
 class Minerva:
     def __init__(self, profile=None):
         kwargs = {}

minerva/pool.py

@@ -1,12 +1,17 @@
 from threading import Thread, Lock
 
 class Pool:
-    def __init__(self, worker, num=1):
+    def __init__(self, worker, num=1, web=False):
         # TODO can move the creation into a thread, but that might be too
         # many concurrent requests for AWS
         self.machines = [worker(i).create() for i in range(num)]
         self.mutex    = None
 
+        if web:
+            import minerva.web
+            minerva.web.POOL = self
+            minerva.web.run()
+
         for machine in self.machines:
             machine.join()
             machine.login()

minerva/web.py

@@ -0,0 +1,17 @@
+from flask import Flask, render_template
+
+POOL = None
+app = Flask(__name__)
+
+@app.get('/')
+def index():
+    return "hello world"
+
+@app.get('/instances')
+def instances():
+    iss = POOL.machines
+    return f"{len(iss)} instances"
+
+def run():
+    app.run(host="0.0.0.0", port=8080, debug=True)
+