From c20ec3e4961846cdb3fbbace2f84dac494ca3f0d Mon Sep 17 00:00:00 2001 From: j1z0 Date: Fri, 9 Jun 2017 15:59:53 -0400 Subject: [PATCH 1/6] ability to pass in path of requirements file --- aws_lambda/aws_lambda.py | 33 +++++++++------------------------ scripts/lambda | 2 +- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py index 7a4eeecf..2e2c01ac 100755 --- a/aws_lambda/aws_lambda.py +++ b/aws_lambda/aws_lambda.py @@ -3,7 +3,6 @@ import json import logging import os -import sys import time from imp import load_source from shutil import copy, copyfile @@ -111,12 +110,6 @@ def invoke(src, alt_event=None, verbose=False): path_to_event_file = os.path.join(src, 'event.json') event = read(path_to_event_file, loader=json.loads) - #Tweak to allow module to import local modules - try: - sys.path.index(src) - except: - sys.path.append(src) - handler = cfg.get('handler') # Inspect the handler string (.) and translate it # into a function we can execute. @@ -149,10 +142,8 @@ def init(src, minimal=False): for filename in os.listdir(templates_path): if (minimal and filename == 'event.json') or filename.endswith('.pyc'): continue - dest_path = os.path.join(templates_path, filename) - - if not os.path.isdir(dest_path): - copy(dest_path, src) + destination = os.path.join(templates_path, filename) + copy(destination, src) def build(src, requirements=False, local_package=None): @@ -185,13 +176,6 @@ def build(src, requirements=False, local_package=None): requirements=requirements, local_package=local_package) - # Hack for Zope. - if "zope" in os.listdir(path_to_temp): - print("Zope packages detected; fixing Zope package paths to make them importable.") - # Touch. - with open(os.path.join(path_to_temp, "zope/__init__.py"), "wb"): - pass - # Gracefully handle whether ".zip" was included in the filename or not. output_filename = ('{0}.zip'.format(output_filename) if not output_filename.endswith('.zip') @@ -204,7 +188,6 @@ def build(src, requirements=False, local_package=None): continue if filename == 'config.yaml': continue - print("Bundling: %r" % filename) files.append(os.path.join(src, filename)) # "cd" into `temp_path` directory. @@ -272,7 +255,7 @@ def _filter_blacklist(package): package = package.replace('-e ', '') print('Installing {package}'.format(package=package)) - pip.main(['install', package, '-t', path, '--ignore-installed']) + pip.main(['install', package, '-t', path]) def pip_install_to_target(path, requirements=False, local_package=None): @@ -300,15 +283,17 @@ def pip_install_to_target(path, requirements=False, local_package=None): print('Gathering requirement packages') data = read("requirements.txt") packages.extend(data.splitlines()) + elif os.path.exists(requirements): + print('Gathering requirement from %s' % requirements) + data = read(requirements) + packages.extend(data.splitlines()) if not packages: print('No dependency packages installed!') if local_package is not None: - if not isinstance(local_package, (list, tuple) ): - local_package = [local_package] - for l_package in local_package: - packages.append(l_package) + # TODO: actually sdist is probably bettter here... + packages.append(local_package) _install_packages(path, packages) diff --git a/scripts/lambda b/scripts/lambda index ad50d5e4..d3d8e5a2 100755 --- a/scripts/lambda +++ b/scripts/lambda @@ -41,7 +41,7 @@ def invoke(event_file, verbose): @click.command(help="Register and deploy your code to lambda.") -@click.option('--use-requirements', default=False, is_flag=True, help='Install all packages defined in requirements.txt') +@click.option('--use-requirements', default=None, help='Install all packages defined in requirements.txt', type=click.Path()) @click.option('--local-package', default=None, help='Install local package as well.', type=click.Path(), multiple=True) def deploy(use_requirements, local_package): aws_lambda.deploy(CURRENT_DIR, use_requirements, local_package) From ad93394cb3ec7d913df820ce27d5fb8fdd5f2509 Mon Sep 17 00:00:00 2001 From: j1z0 Date: Fri, 9 Jun 2017 16:11:13 -0400 Subject: [PATCH 2/6] maybe I'll put this back in --- aws_lambda/aws_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py index 2e2c01ac..1a46d8bb 100755 --- a/aws_lambda/aws_lambda.py +++ b/aws_lambda/aws_lambda.py @@ -255,7 +255,7 @@ def _filter_blacklist(package): package = package.replace('-e ', '') print('Installing {package}'.format(package=package)) - pip.main(['install', package, '-t', path]) + pip.main(['install', package, '-t', path, '--ignore-installed']) def pip_install_to_target(path, requirements=False, local_package=None): From e980edcbf4197da27b10ae8105e6eacedb02f3da Mon Sep 17 00:00:00 2001 From: j1z0 Date: Tue, 8 Aug 2017 15:10:04 -0400 Subject: [PATCH 3/6] added the raw copy parameter --- aws_lambda/aws_lambda.py | 25 ++++++++++++++++++++++--- scripts/lambda | 1 + 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py index 1a46d8bb..e6103025 100755 --- a/aws_lambda/aws_lambda.py +++ b/aws_lambda/aws_lambda.py @@ -62,7 +62,7 @@ def cleanup_old_versions(src, keep_last_versions): .format(version_number, e.message)) -def deploy(src, requirements=False, local_package=None): +def deploy(src, requirements=False, local_package=None, raw_copy=None): """Deploys a new function to AWS Lambda. :param str src: @@ -80,7 +80,7 @@ def deploy(src, requirements=False, local_package=None): # folder then add the handler file in the root of this directory. # Zip the contents of this folder into a single file and output to the dist # directory. - path_to_zip_file = build(src, requirements, local_package) + path_to_zip_file = build(src, requirements, local_package, raw_copy) if function_exists(cfg, cfg.get('function_name')): update_function(cfg, path_to_zip_file) @@ -146,7 +146,7 @@ def init(src, minimal=False): copy(destination, src) -def build(src, requirements=False, local_package=None): +def build(src, requirements=False, local_package=None, raw_copy=None): """Builds the file bundle. :param str src: @@ -176,6 +176,25 @@ def build(src, requirements=False, local_package=None): requirements=requirements, local_package=local_package) + if raw_copy: + raw_files = [] + for filename in os.listdir(raw_copy): + if os.path.isfile(filename): + if filename == '.DS_Store': + continue + if filename == 'config.yaml': + continue + raw_files.append(os.path.join(src, filename)) + + # "cd" into `temp_path` directory. + os.chdir(path_to_temp) + for f in raw_files: + _, filename = os.path.split(f) + + # Copy handler file into root of the packages folder. + copyfile(f, os.path.join(path_to_temp, filename)) + + # Gracefully handle whether ".zip" was included in the filename or not. output_filename = ('{0}.zip'.format(output_filename) if not output_filename.endswith('.zip') diff --git a/scripts/lambda b/scripts/lambda index d3d8e5a2..42892510 100755 --- a/scripts/lambda +++ b/scripts/lambda @@ -29,6 +29,7 @@ def init(folder): @click.command(help="Bundles package for deployment.") @click.option('--use-requirements', default=False, is_flag=True, help='Install all packages defined in requirements.txt') @click.option('--local-package', default=None, help='Install local package as well.', type=click.Path(), multiple=True) +@click.option('--raw-copy', default=None, help='directory of stuff to raw copy to the lambda dir', type=click.Path(), multiple=True) def build(use_requirements, local_package): aws_lambda.build(CURRENT_DIR, use_requirements, local_package) From 14857bc59b6de1562cfbc4439bed37baf0094ba6 Mon Sep 17 00:00:00 2001 From: j1z0 Date: Sun, 22 Oct 2017 12:31:01 -0400 Subject: [PATCH 4/6] support for embedding binaries --- aws_lambda/aws_lambda.py | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py index e6103025..183344e1 100755 --- a/aws_lambda/aws_lambda.py +++ b/aws_lambda/aws_lambda.py @@ -166,34 +166,22 @@ def build(src, requirements=False, local_package=None, raw_copy=None): path_to_dist = os.path.join(src, dist_directory) mkdir(path_to_dist) + path_to_temp = mkdtemp(prefix='aws-lambda') + + if raw_copy: + bin_dir = os.path.join(path_to_temp, "lambda_bin") + import shutil + shutil.copytree(raw_copy, bin_dir) + # Combine the name of the Lambda function with the current timestamp to use # for the output filename. function_name = cfg.get('function_name') output_filename = "{0}-{1}.zip".format(timestamp(), function_name) - path_to_temp = mkdtemp(prefix='aws-lambda') pip_install_to_target(path_to_temp, requirements=requirements, local_package=local_package) - if raw_copy: - raw_files = [] - for filename in os.listdir(raw_copy): - if os.path.isfile(filename): - if filename == '.DS_Store': - continue - if filename == 'config.yaml': - continue - raw_files.append(os.path.join(src, filename)) - - # "cd" into `temp_path` directory. - os.chdir(path_to_temp) - for f in raw_files: - _, filename = os.path.split(f) - - # Copy handler file into root of the packages folder. - copyfile(f, os.path.join(path_to_temp, filename)) - # Gracefully handle whether ".zip" was included in the filename or not. output_filename = ('{0}.zip'.format(output_filename) From cfade12ea72146f7ce4aa91ef77736981f7ae79c Mon Sep 17 00:00:00 2001 From: j1z0 Date: Mon, 29 Jan 2018 16:27:08 -0500 Subject: [PATCH 5/6] for whatever reason some times list_functions doesn't get all functions, whereas get_function by name seems to work more consistently --- aws_lambda/aws_lambda.py | 10 +++++----- scripts/lambda | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py index 183344e1..86708912 100755 --- a/aws_lambda/aws_lambda.py +++ b/aws_lambda/aws_lambda.py @@ -405,8 +405,8 @@ def function_exists(cfg, function_name): aws_secret_access_key = cfg.get('aws_secret_access_key') client = get_client('lambda', aws_access_key_id, aws_secret_access_key, cfg.get('region')) - functions = client.list_functions().get('Functions', []) - for fn in functions: - if fn.get('FunctionName') == function_name: - return True - return False + try: + client.get_function(FunctionaName=function_name) + except: + return False + return True diff --git a/scripts/lambda b/scripts/lambda index 42892510..6af24019 100755 --- a/scripts/lambda +++ b/scripts/lambda @@ -30,8 +30,8 @@ def init(folder): @click.option('--use-requirements', default=False, is_flag=True, help='Install all packages defined in requirements.txt') @click.option('--local-package', default=None, help='Install local package as well.', type=click.Path(), multiple=True) @click.option('--raw-copy', default=None, help='directory of stuff to raw copy to the lambda dir', type=click.Path(), multiple=True) -def build(use_requirements, local_package): - aws_lambda.build(CURRENT_DIR, use_requirements, local_package) +def build(use_requirements, local_package, raw_copy): + aws_lambda.build(CURRENT_DIR, use_requirements, local_package, raw_copy) @click.command(help="Run a local test of your function.") From f90b057d59a49e165bbd15cc94d93f54a1121abd Mon Sep 17 00:00:00 2001 From: j1z0 Date: Thu, 8 Feb 2018 10:26:47 -0500 Subject: [PATCH 6/6] mis spelling --- aws_lambda/aws_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aws_lambda/aws_lambda.py b/aws_lambda/aws_lambda.py index 86708912..23362b51 100755 --- a/aws_lambda/aws_lambda.py +++ b/aws_lambda/aws_lambda.py @@ -406,7 +406,7 @@ def function_exists(cfg, function_name): client = get_client('lambda', aws_access_key_id, aws_secret_access_key, cfg.get('region')) try: - client.get_function(FunctionaName=function_name) + client.get_function(FunctionName=function_name) except: return False return True