"""
Define common admin and maintenance tasks here.
For more info: http://docs.fabfile.org/en/latest/
"""
import sys
from path import path
from fabric.api import run, env, prefix, quiet, abort
PROJECT_ROOT = path(__file__).abspath().realpath().dirname()
sys.path.append(PROJECT_ROOT / 'setup')
from fabutils import conf
conf.configure(PROJECT_ROOT, 'msgvis')
from fabutils import factories
from fabutils.tasks import *
pip_requirements = {
'dev': ('-r requirements/dev.txt',),
'prod': ('-r requirements/prod.txt',),
'test': ('-r requirements/test.txt',),
}
required_nltk_corpora = ["stopwords", "punkt", "wordnet"]
pip_install = factories.pip_install_task(pip_requirements, default_env='dev')
[docs]def dependencies(default_env='dev'):
"""Install requirements for pip, npm, and bower all at once."""
pip_install(default_env)
npm_install()
bower_install()
nltk_init()
test = factories.test_task(default_settings='msgvis.settings.test')
test_coverage = factories.coverage_task(default_settings='msgvis.settings.test')
test_data_path = conf.PROJECT_ROOT / 'setup' / 'fixtures' / 'test_data.json'
make_test_data = factories.make_test_data_task(('base', 'api', # 'corpus',
'dimensions', 'datatable',
'importer', 'enhance', 'questions',
'auth', '--exclude=auth.Permission'),
test_data_path)
load_test_data = factories.load_test_data_task(test_data_path)
# Model keys to fixture paths from PROJECT_ROOT
model_fixtures = (
('corpus.Language', 'msgvis/apps/corpus/fixtures/languages.json'),
('corpus.MessageType', 'msgvis/apps/corpus/fixtures/messagetypes.json'),
('corpus.Timezone', 'msgvis/apps/corpus/fixtures/timezones.json'),
('dimensions', 'msgvis/apps/dimensions/fixtures/dimensions.json'),
('questions', 'msgvis/apps/questions/fixtures/questions.json'),
)
[docs]def generate_fixtures():
"""
Regenerate configured fixtures from the database.
"""
generated = []
for model, fixturefile in model_fixtures:
fabutils.manage_py('dumpdata --indent=2 {model} > {out}'.format(
model=model,
out=PROJECT_ROOT / fixturefile,
))
generated.append(fixturefile)
print "Generated %d fixtures:" % len(generated)
if len(generated) > 0:
print " - " + '\n - '.join(generated)
[docs]def load_fixtures():
"""
Replaces the database tables with the contents of fixtures.
"""
for model, fixturefile in model_fixtures:
fabutils.manage_py('syncdata %s' % (PROJECT_ROOT / fixturefile,))
[docs]def import_corpus(dataset_file_or_dir):
"""Import a dataset from a file"""
dataset_file_or_dir = path(dataset_file_or_dir).abspath().realpath()
fabutils.manage_py('import_corpus %s' % dataset_file_or_dir)
[docs]def restart_webserver():
"""Restart a local gunicorn process"""
print green("Restarting gunicorn...")
fabutils.manage_py('supervisor restart webserver')
[docs]def supervisor():
"""Starts the supervisor process"""
print green("Supervisor launching...")
fabutils.manage_py('supervisor')
[docs]def deploy(branch=None):
"""
SSH into a remote server, run commands to update deployment,
and start the server.
This requires that the server is already running a
fairly recent copy of the code.
Furthermore, the app must use a
"""
denv = fabutils.dot_env()
host = denv.get('DEPLOY_HOST', None)
virtualenv = denv.get('DEPLOY_VIRTUALENV', None)
if host is None:
print red("No DEPLOY_HOST in .env file")
return
if virtualenv is None:
print red("No DEPLOY_VIRTUALENV in .env file")
return
env.host_string = host
with prefix('workon %s' % virtualenv):
run('which python')
# Check prereqs
with quiet():
pips = run('pip freeze')
if "Fabric" not in pips or 'path.py' not in pips:
print green("Installing Fabric...")
run('pip install -q Fabric path.py')
run('git pull')
if branch:
run('git checkout %s' % branch)
run('git pull')
run('fab dependencies:prod')
run('fab print_env check_database migrate')
run('fab build_static restart_webserver')
[docs]def topic_pipeline(dataset, name="my topic model", num_topics=30):
"""Run the topic pipeline on a dataset"""
command = "extract_topics --topics %d --name '%s' %s" % (int(num_topics), name, dataset)
fabutils.manage_py(command)
[docs]def info():
"""Print a bunch of info about the environment"""
fabutils.env_info()
print os.linesep, green("---------- Scientific computing packages ------------"), os.linesep
fabutils.try_load_module('nltk')
fabutils.try_load_module('gensim')
numpy = fabutils.try_load_module('numpy')
if numpy is not None:
print " Numpy sysinfo:"
import numpy.distutils.system_info as sysinfo
print " lapack:", sysinfo.get_info('lapack')
print " blas:", sysinfo.get_info('blas')
print " atlas:", sysinfo.get_info('atlas')
nltk_init = factories.nltk_download_task(required_nltk_corpora)
[docs]def memcached_status():
"""Display the status of the memcached server"""
denv = fabutils.dot_env(default={})
if denv.get('MEMCACHED_LOCATION', None) is not None:
local("watch -n1 -d 'memcstat --servers %s'" % denv.get('MEMCACHED_LOCATION', None))
else:
print yellow("Set MEMCACHED_LOCATION in .env")