Date post: | 15-Apr-2017 |
Category: |
Engineering |
Upload: | simone-federici |
View: | 293 times |
Download: | 0 times |
Fabric al PyCon 5 di FirenzeTaa taa chi trova fabric non lo lascia più…
$ fab taskA taskB
from fabric.api import run, env
env.hosts = ['host1', ‘host2']
def taskA(): run(‘ls')
def taskB(): run('whoami')
$ fab mytask:roles=role1
from fabric.api import env
env.roledefs = { 'web': ['www1', 'www2', 'www3'], 'dns': ['ns1', ‘ns2']
}
def mytask(): run('ls /var/www')
$ fab mytask:roles=role1,exclude_hosts="a;c"
from fabric.api import env, hosts, roles, run
env.roledefs = {'role1': ['b', ‘c']}
@hosts('a', ‘b') @roles(‘role1') def mytask():
run('ls /var/www')
$ fab migrate update
from fabric.api import run, roles
env.roledefs = { 'db': ['db1', 'db2'], 'web': ['web1', 'web2', ‘web3'],
}
@roles(‘db') def migrate():
# Database stuff here. pass
@roles(‘web') def update():
# Code updates here. pass
fab deploy
from fabric.api import run, roles, execute
def deploy(): execute(migrate) execute(update)
migrate on db1 migrate on db2 update on web1 update on web2 update on web3
$ fab deploy:app or $ fab deploy:db
from fabric.api import run, execute, task
from mylib import external_datastore
def do_work(): run("something interesting on a host”)
@task def deploy(lookup_param):
host_list = external_datastore.query(lookup_param) execute(do_work, hosts=host_list)
$ fab set_hosts:app do_work
from fabric.api import run, task from mylib import external_datastore
@task def do_work():
run("something interesting on a host")
@task def set_hosts(lookup_param):
env.hosts = external_datastore.query(lookup_param)
Combining stdout and stderr
run("cmd", pty=False, combine_stderr=True):
run("cmd", pty=False, combine_stderr=False):
run("cmd", pty=True, combine_stderr=False):
$ fab -H host1,host2,host3 runs_in_parallel runs_seriallyfrom fabric.api import *
@parallel def runs_in_parallel():
pass
def runs_serially(): pass
runs_in_parallel on host1, host2, and host3 runs_serially on host1 runs_serially on host2 runs_serially on host3
$ fab -P -z 5 heavy_task
from fabric.api import *
@parallel(pool_size=5) def heavy_task(): # lots of heavy local lifting or lots of IO here
@task(alias=’short’)
from fabric.api import task, run
@task def mytask():
run("a command")
@task(alias=‘dwm’) def deploy_with_migrations():
pass
Submodule deploy.py
@task(default=True) def full_deploy():
pass
$ fab --list Available commands:
deploy deploy.full_deploy deploy.migrate
Class Task
class MyTask(Task): name = "deploy" def run(self, environment, domain="whatever.com"): run("git clone foo") sudo("service apache2 restart")
instance = MyTask()
VS
@task def deploy(environment, domain="whatever.com"):
run("git clone foo") sudo("service apache2 restart")
Colors
from fabric.colors import green print(green("This text is green!"))
fabric.colors.blue(text, bold=False) fabric.colors.cyan(text, bold=False) fabric.colors.green(text, bold=False) fabric.colors.magenta(text, bold=False) fabric.colors.red(text, bold=False) fabric.colors.white(text, bold=False) fabric.colors.yellow(text, bold=False)
Context managersdef mytask():
with cd('/path/to/app'), prefix('workon myvenv'): run('./manage.py syncdb') run('./manage.py loaddata myfixture') with cd('/var/www'): run('ls') # cd /var/www && ls with cd('website1'): run('ls') # cd /var/www/website1 && ls with hide('running', 'stdout', 'stderr'): run('ls /var/www') # Map localhost:6379 on the server to localhost:6379 on the client, # so that the remote 'redis-cli' program ends up speaking to the local # redis-server. with remote_tunnel(6379): run("redis-cli -i")
Contrib
Django Integration Rsync Project Upload Project Console Confirm y/n Files and Directory
So… what is Fabric?
● Deploy ● Manage multiple server ● Clustering ● Multiplatform ● Parallel ● Testing ● SSH Authentication
SSH + Bash Power + Python = Rocks**3 = Fabric
fab release:master
…
django integration
from fabric.contrib import django
django.settings_module(‘myproject.settings')
from django.conf import settings
def dump_production_database(): run(pg_dump -U %s -w %s > /bck/prod-db.sql' % ( settings.DATABASE_USER, settings.DATABASE_NAME ))
fab deploy
@hosts('[email protected]')def deploy():
with cd("/opt/myproject"): run("git pull") run("django-admin.py collectstatic --noinput") run("django-admin.py migrate --noinput") run("/etc/init.d/uwsgi stop || echo 'done'") run("/etc/init.d/uwsgi start")
fab backup_and_publish
@task@hosts('[email protected]')def backup_and_publish(): run('''tar cjvf /var/wwwbackups/www.%s.tar.bz2
--exclude=/var/www/download —exclude=/var/www/backup* /var/www''' % today().strftime("%Y%m%d%H%M%S")) run('rsync -avz --checksum --ignore-times
/var/wwwstage/ /var/www') #--delete
fab static_generation
@taskdef static_generation():
execute(remote_generation) local("wget --user=admin --password=rootme --recursive --page-requisites --html-extension --convert-links --restrict-file-names=windows --domains example.com --no-parent http://wwwstage.example.com/ -o dump.log || echo 'Looking for 404 on wwwstage.example.com'") local("cat dump.log | grep -B 2 '404 Not Found' | grep 'http://wwwstage.example.com/' || echo 'OK no 404 found...'")
fab upload_release_note:2.10.0
@task@hosts('[email protected]')def upload_release_note(version):
release_note_file_name = "RELEASE_NOTE_%s.TXT" % version with open(release_note_file_name,"w") as out_file: notes = jira.Query().render(version=version) out_file.write(notes.encode('ascii', 'ignore')) out_file.close() put(release_note_file_name, "/cygdrive/d/%s/" % version)
fab cleanup_nexus:10.0.2-RC2
@taskdef cleanup_nexus(version):
for module in [ "core-api", "core-client-rest", "core-manual", "core-web"]:
local("curl -X DELETE -u user:rootme http://nexus.example.com:8180/nexus/service/local/repositories/releases/content/example/%s/%s/" % (module, version))
LOCK_FILE = “~/.lockfile.release.core.lock"
class Lock(): def __enter__(self): if os.access(os.path.expanduser(LOCK_FILE), os.F_OK): pidfile = open(os.path.expanduser(LOCK_FILE), "r") pidfile.seek(0) old_pid = pidfile.readline() print "There is an already a process running with pid: %s," % old_pid sys.exit(1) pidfile = open(os.path.expanduser(LOCK_FILE), "w") pidfile.write("%s" % os.getpid()) pidfile.close def __exit__(self, type, value, traceback): os.remove(os.path.expanduser(LOCK_FILE))
Locks
fab send_email_candidate:2.12,me@...
@taskdef send_mail_candidate(version, *receivers):
sender = '[email protected]' body = """From: Core Team <[email protected]>To: Development <[email protected]>Subject: New Release CANDIDATE %(version)s\nNew Release CANDIDATE %(version)savailable on: * smb://example.com/myproject/%(version)s """ % dict(version=version) try: message = smtplib.SMTP('example.com') message.sendmail(sender, receivers, body) print "Successfully sent email" except smtplib.SMTPException: print "Error: unable to send email"
XML parser@taskdef get_pom_version():
src=os.path.dirname(__file__) pom_file=os.path.abspath(os.path.join(src, 'pom.xml')) from xml.dom.minidom import parse pom = parse(pom_file) version = pom.getElementsByTagName("version")[1].firstChild.nodeValue find = re.compile(r'^\d+.\d+.\d+-([a-zA-Z-]+)\d*-SNAPSHOT$').findall(version) if not find: abort(version + " is not a valid development version") versions = re.compile(r'\d+').findall(version) if len(versions) is 3: versions.append(1) if len(versions) is not 4: abort(version + " is not a valid development version") versions.extend(find) return versions
fab sed_poms_version:2.13
@taskdef sed_poms_version(new_version):
major, minor, patch, rev, rc = get_pom_version()
version = "%s.%s.%s-%s%s-SNAPSHOT" % (major, minor, patch, rc, rev)
local("sed -i '' 's@%s@%s@g' pom.xml */pom.xml" % (version, new_version))
fab create_manuals:2.10
ACTIVATE="source /home/installer/.sphinx_env/bin/activate"@hosts('[email protected]')def create_manuals(version): with cd(DEPLOY_DIR + "/myproject"): name = "manual-%s-doc" % version run("wget http://nexus.example.com:8180/nexus/service/local/repo_groups/public/content/com/myproject/manual/%s/%s.zip" % (version, name)) with cd(name): run(ACTIVATE + "&& make docs") run("mv target/en/latex/MyProject*.pdf docs/" % version) run("tar cjvf docs/MyCDDS-en-%s.tar.bz2 target/docs/" % version) run("scp -r docs [email protected]:/cygdrive/d/%s/docs" % version)
fab create_installer:2.10 (innosetup)
@hosts('[email protected]')def create_installer(version): name = "installer-%s-app" % version run("wget http://nexus.geniusbytes.com:8180/nexus/service/local/repo_groups/public/content/myproject/installer/%s/%s.zip" % (version, name)) with cd(name): run("tar xjvf /cygdrive/d/%(version)s/MyCDDS-en-%(version)s.tar.bz2" % dict(version=version)) run(ANT) run("mkdir -p /cygdrive/d/%s" % version) run("cp build/*.exe /cygdrive/d/%s/" % version) run("cp build-update/*.exe /cygdrive/d/%s/" % version) run("rm -rf %s" % name)
fab release_core:master,2.10,2.11
@hosts('[email protected]')def release_core(branch, version, next_version):
with cd(CORE_DEPLOY_DIR): run('git clone ssh://[email protected]/srv/git/myproject.git') with cd(CORE_DEPLOY_DIR + "/myproject"): run("git checkout %s" % branch) run("mvn clean install") run("mvn --batch-mode release:clean release:prepare
-DreleaseVersion=%s -DdevelopmentVersion=%s" % (version, next_version))
run("mvn release:perform")
fab release:master
def release(branch): with Lock(): major, minor, patch, rev, rc = check_current_version(branch) if 'RC' != rc: abort("RC not found, not possible release a final version") version = "%s.%s.%s" % (major, minor, patch) next_version = "%s.%s.%s-RC%s-SNAPSHOT" % (major, minor, int(patch)+1, 1) puts("preparing to release %s (next will be %s)" % (version, next_version)) execute(release_core, branch, version, next_version) execute(create_manuals, version) execute(create_installer, version) execute(upload_release_note, version) execute(send_mail_final_release, version, '[email protected]', '[email protected]') local("git pull") execute(labels.missing_translations, '[email protected]')
fab cluod_remote_control
...
EC2 Testing with 200 micro server
EC2 + Fabric + Funkload
● EC2 use all ubuntu standard AMI ● Fabric as remote control, move files, aggregate. ● Funkload in order to stress an internet application. (not on
EC2)
Performance Testing Architecture
Target
Cloud CTRL
Tester
TargetTarget
FunkLoad
Fabric (nmon+pefmon)
Fabric + EC2
Fabric + EC2
Testing phases
I. Prepare Monitoring II.Prepare Cloud
1. Start Monitoring 2. Start Parallel Testing 3. Collecting Test Results 4. Collecting Perf Results 5. Reporting
Target
Cloud CTRL
Tester
TargetTarget
FunkLoad
Fabric (nmon+pefmon)
Fabric + EC2
Fabric + EC2
Testing Console
fab prepare_monitoring fab prepare_cloud
fab start_monitoring fab start_testing:ciccio,100,5000 fab collecting_test_results:ciccio fab collecting_perf_results:ciccio fab reporting:ciccio