diff --git a/examples/repartition.py b/examples/repartition.py index 093a0b1..84fb52b 100644 --- a/examples/repartition.py +++ b/examples/repartition.py @@ -47,7 +47,7 @@ def repartition(mach, agents): ##################################### # Prep the work # Find out how many hours there are in the dataset -pool_size = 1 +pool_size = 5 objs = s.m.s3.ls(src_top_level + "year=") hours = set(["s3://" + '/'.join([o.bucket_name, *o.key.split("/")[0:-1]]) @@ -66,7 +66,7 @@ try: ####################################### # Create the machines # This also waits for them to be made - pool = m.Pool(s.worker, pool_size) + pool = m.Pool(s.worker, pool_size, web=True) ######################################## # Now that we have the pool, put them to work @@ -74,10 +74,13 @@ try: # doing that until the list is empty # First part: sort the individual files - pool.run(sort_hour, data=hours) + #pool.run(sort_hour, data=hours) # Second part: repartition - pool.run(repartition, data=groups) + #pool.run(repartition, data=groups) + + import IPython + IPython.embed() finally: pool.terminate() diff --git a/minerva/minerva.py b/minerva/minerva.py index d0e9e33..8e1203c 100644 --- a/minerva/minerva.py +++ b/minerva/minerva.py @@ -1,11 +1,6 @@ import boto3 import minerva as m -class Minerva: - def __init__(self, profile): - self.session = boto3.session.Session(profile_name=profile) - self.s3 = m.S3(self) - class Minerva: def __init__(self, profile=None): kwargs = {} diff --git a/minerva/pool.py b/minerva/pool.py index 3096a66..5b58e24 100644 --- a/minerva/pool.py +++ b/minerva/pool.py @@ -1,12 +1,17 @@ from threading import Thread, Lock class Pool: - def __init__(self, worker, num=1): + def __init__(self, worker, num=1, web=False): # TODO can move the creation into a thread, but that might be too # many concurrent requests for AWS self.machines = [worker(i).create() for i in range(num)] self.mutex = None + if web: + import minerva.web + minerva.web.POOL = self + minerva.web.run() + for machine in self.machines: machine.join() machine.login() diff --git a/minerva/web.py b/minerva/web.py new file mode 100644 index 0000000..1a7f507 --- /dev/null +++ b/minerva/web.py @@ -0,0 +1,17 @@ +from flask import Flask, render_template + +POOL = None +app = Flask(__name__) + +@app.get('/') +def index(): + return "hello world" + +@app.get('/instances') +def instances(): + iss = POOL.machines + return f"{len(iss)} instances" + +def run(): + app.run(host="0.0.0.0", port=8080, debug=True) +