From d523c83d689f0be9bf2f8aab0c5349ac0475ce99 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 11:57:06 -0400 Subject: [PATCH 01/14] Added ability to use a custom loader in run_algorithm() --- .vscode/settings.json | 3 +++ zipline/utils/run_algo.py | 51 +++++++++++++++++++++------------------ 2 files changed, 31 insertions(+), 23 deletions(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000..77c68caca4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "python.pythonPath": "/Users/calmitchell/.pyenv/versions/3.5.5/bin/python" +} \ No newline at end of file diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index db33b549bb..8a86e0b3cd 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -52,26 +52,28 @@ def __str__(self): return self.pyfunc_msg -def _run(handle_data, - initialize, - before_trading_start, - analyze, - algofile, - algotext, - defines, - data_frequency, - capital_base, - data, - bundle, - bundle_timestamp, - start, - end, - output, - trading_calendar, - print_algo, - metrics_set, - local_namespace, - environ): +def _run( + my_loader, + handle_data, + initialize, + before_trading_start, + analyze, + algofile, + algotext, + defines, + data_frequency, + capital_base, + data, + bundle, + bundle_timestamp, + start, + end, + output, + trading_calendar, + print_algo, + metrics_set, + local_namespace, + environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. @@ -168,9 +170,10 @@ def _run(handle_data, def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader - raise ValueError( + return my_loader + """raise ValueError( "No PipelineLoader registered for column %s." % column - ) + )""" else: env = TradingEnvironment(environ=environ) choose_loader = None @@ -285,7 +288,8 @@ def run_algorithm(start, default_extension=True, extensions=(), strict_extensions=True, - environ=os.environ): + environ=os.environ, + my_loader=None): """Run a trading algorithm. Parameters @@ -376,6 +380,7 @@ def run_algorithm(start, ) return _run( + my_loader, handle_data=handle_data, initialize=initialize, before_trading_start=before_trading_start, From 2a8018b383e02924fb90740dd2450f24b522526f Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:34:33 -0400 Subject: [PATCH 02/14] run_algorithm() can now accept an optional dictionary of PipelineLoaders --- zipline/utils/run_algo.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 8a86e0b3cd..2161ac1843 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -53,7 +53,7 @@ def __str__(self): def _run( - my_loader, + loaders, handle_data, initialize, before_trading_start, @@ -170,10 +170,12 @@ def _run( def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader - return my_loader - """raise ValueError( - "No PipelineLoader registered for column %s." % column - )""" + elif column in loaders: + return loaders[column] + else: + raise ValueError( + "No PipelineLoader registered for column %s." % column + ) else: env = TradingEnvironment(environ=environ) choose_loader = None @@ -289,7 +291,7 @@ def run_algorithm(start, extensions=(), strict_extensions=True, environ=os.environ, - my_loader=None): + loaders=None): """Run a trading algorithm. Parameters @@ -348,6 +350,9 @@ def run_algorithm(start, environ : mapping[str -> str], optional The os environment to use. Many extensions use this to get parameters. This defaults to ``os.environ``. + loaders : iterable{PipelineLoader}, optional + A dictionary containing custom loaders to include data from external sources + in a pipeline. Returns ------- @@ -380,7 +385,6 @@ def run_algorithm(start, ) return _run( - my_loader, handle_data=handle_data, initialize=initialize, before_trading_start=before_trading_start, @@ -401,4 +405,5 @@ def run_algorithm(start, metrics_set=metrics_set, local_namespace=False, environ=environ, + loaders=loaders ) From d1e96b4e1403923613c48b6eef89440f7ae598db Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:38:44 -0400 Subject: [PATCH 03/14] ignoring vscode setting json file --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index d0cd8e131a..934fd19861 100644 --- a/.gitignore +++ b/.gitignore @@ -76,3 +76,5 @@ TAGS .ipynb_checkpoints/ .gdb_history + +.vscode From bc0ca30e115630b930cbf3aab6bf9e9114ed348f Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:41:52 -0400 Subject: [PATCH 04/14] removed vscode json settings --- .vscode/settings.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 77c68caca4..0000000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "python.pythonPath": "/Users/calmitchell/.pyenv/versions/3.5.5/bin/python" -} \ No newline at end of file From 4579095c4c8ebbfb6429692a9b0aba7a422e7315 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:43:36 -0400 Subject: [PATCH 05/14] ignore gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 934fd19861..0bb413d4ee 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,5 @@ TAGS .gdb_history .vscode + +.gitignore \ No newline at end of file From c4dd2800de4e006aef9004d719e7e6a995de7e40 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:44:02 -0400 Subject: [PATCH 06/14] removed gitignore --- .gitignore | 82 ------------------------------------------------------ 1 file changed, 82 deletions(-) delete mode 100644 .gitignore diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 0bb413d4ee..0000000000 --- a/.gitignore +++ /dev/null @@ -1,82 +0,0 @@ -.bundle -db/*.sqlite3 -log/*.log -*.log -tmp/**/* -tmp/* -*.swp -*~ -#mac autosaving file -.DS_Store -*.py[co] - -# Installer logs -pip-log.txt - -# Unit test / coverage reports -.coverage -.tox -test.log -.noseids -*.xlsx - -# Compiled python files -*.py[co] - -# Packages -*.egg -.eggs/* -*.egg-info -dist -build -eggs -cover -parts -bin -var -sdist -develop-eggs -.installed.cfg -coverage.xml -htmlcov -nosetests.xml - -# C Extensions -*.o -*.so -*.out -# git add -f if needed -*.c - -# Vim -*.swp -*.swo - -# Built documentation -docs/_build/* - -# Un-tarred example data input. We should only commit the tarball. -tests/resources/example_data/* - -# database of vbench -benchmarks.db - -# Vagrant temp folder -.vagrant - -# Intellij IDE temp project files -.project -zipline.iml - -# PyCharm custom settings -.idea - -TAGS - -.ipynb_checkpoints/ - -.gdb_history - -.vscode - -.gitignore \ No newline at end of file From db45582d659019d4017e7b81894f757b57450473 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:44:47 -0400 Subject: [PATCH 07/14] added gitignore back --- .gitignore | 80 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..934fd19861 --- /dev/null +++ b/.gitignore @@ -0,0 +1,80 @@ +.bundle +db/*.sqlite3 +log/*.log +*.log +tmp/**/* +tmp/* +*.swp +*~ +#mac autosaving file +.DS_Store +*.py[co] + +# Installer logs +pip-log.txt + +# Unit test / coverage reports +.coverage +.tox +test.log +.noseids +*.xlsx + +# Compiled python files +*.py[co] + +# Packages +*.egg +.eggs/* +*.egg-info +dist +build +eggs +cover +parts +bin +var +sdist +develop-eggs +.installed.cfg +coverage.xml +htmlcov +nosetests.xml + +# C Extensions +*.o +*.so +*.out +# git add -f if needed +*.c + +# Vim +*.swp +*.swo + +# Built documentation +docs/_build/* + +# Un-tarred example data input. We should only commit the tarball. +tests/resources/example_data/* + +# database of vbench +benchmarks.db + +# Vagrant temp folder +.vagrant + +# Intellij IDE temp project files +.project +zipline.iml + +# PyCharm custom settings +.idea + +TAGS + +.ipynb_checkpoints/ + +.gdb_history + +.vscode From ef6c8600d3420c3082f4b3071c78fe5658748a64 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:47:41 -0400 Subject: [PATCH 08/14] changed some formatting --- zipline/utils/run_algo.py | 81 ++++++++++++++++++++------------------- 1 file changed, 42 insertions(+), 39 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 2161ac1843..298c314ded 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -53,27 +53,28 @@ def __str__(self): def _run( - loaders, - handle_data, - initialize, - before_trading_start, - analyze, - algofile, - algotext, - defines, - data_frequency, - capital_base, - data, - bundle, - bundle_timestamp, - start, - end, - output, - trading_calendar, - print_algo, - metrics_set, - local_namespace, - environ): + loaders, + handle_data, + initialize, + before_trading_start, + analyze, + algofile, + algotext, + defines, + data_frequency, + capital_base, + data, + bundle, + bundle_timestamp, + start, + end, + output, + trading_calendar, + print_algo, + metrics_set, + local_namespace, + environ + ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. @@ -274,24 +275,26 @@ def load_extensions(default, extensions, strict, environ, reload=False): _loaded_extensions.add(ext) -def run_algorithm(start, - end, - initialize, - capital_base, - handle_data=None, - before_trading_start=None, - analyze=None, - data_frequency='daily', - data=None, - bundle=None, - bundle_timestamp=None, - trading_calendar=None, - metrics_set='default', - default_extension=True, - extensions=(), - strict_extensions=True, - environ=os.environ, - loaders=None): +def run_algorithm( + start, + end, + initialize, + capital_base, + handle_data=None, + before_trading_start=None, + analyze=None, + data_frequency='daily', + data=None, + bundle=None, + bundle_timestamp=None, + trading_calendar=None, + metrics_set='default', + default_extension=True, + extensions=(), + strict_extensions=True, + environ=os.environ, + loaders=None +): """Run a trading algorithm. Parameters From 3bfeb7bd9f861d986285dced59f0d4310c586035 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:50:44 -0400 Subject: [PATCH 09/14] changed formatting --- zipline/utils/run_algo.py | 125 ++++++++++++++++++-------------------- 1 file changed, 60 insertions(+), 65 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 298c314ded..fc131e281b 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -53,28 +53,27 @@ def __str__(self): def _run( - loaders, - handle_data, - initialize, - before_trading_start, - analyze, - algofile, - algotext, - defines, - data_frequency, - capital_base, - data, - bundle, - bundle_timestamp, - start, - end, - output, - trading_calendar, - print_algo, - metrics_set, - local_namespace, - environ - ): + loaders, + handle_data, + initialize, + before_trading_start, + analyze, + algofile, + algotext, + defines, + data_frequency, + capital_base, + data, + bundle, + bundle_timestamp, + start, + end, + output, + trading_calendar, + print_algo, + metrics_set, + local_namespace, + environ): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. @@ -275,26 +274,24 @@ def load_extensions(default, extensions, strict, environ, reload=False): _loaded_extensions.add(ext) -def run_algorithm( - start, - end, - initialize, - capital_base, - handle_data=None, - before_trading_start=None, - analyze=None, - data_frequency='daily', - data=None, - bundle=None, - bundle_timestamp=None, - trading_calendar=None, - metrics_set='default', - default_extension=True, - extensions=(), - strict_extensions=True, - environ=os.environ, - loaders=None -): +def run_algorithm(start, + end, + initialize, + capital_base, + handle_data=None, + before_trading_start=None, + analyze=None, + data_frequency='daily', + data=None, + bundle=None, + bundle_timestamp=None, + trading_calendar=None, + metrics_set='default', + default_extension=True, + extensions=(), + strict_extensions=True, + environ=os.environ, + loaders=None): """Run a trading algorithm. Parameters @@ -387,26 +384,24 @@ def run_algorithm( 'cannot specify `bundle_timestamp` without passing `bundle`', ) - return _run( - handle_data=handle_data, - initialize=initialize, - before_trading_start=before_trading_start, - analyze=analyze, - algofile=None, - algotext=None, - defines=(), - data_frequency=data_frequency, - capital_base=capital_base, - data=data, - bundle=bundle, - bundle_timestamp=bundle_timestamp, - start=start, - end=end, - output=os.devnull, - trading_calendar=trading_calendar, - print_algo=False, - metrics_set=metrics_set, - local_namespace=False, - environ=environ, - loaders=loaders - ) + return _run(handle_data=handle_data, + initialize=initialize, + before_trading_start=before_trading_start, + analyze=analyze, + algofile=None, + algotext=None, + defines=(), + data_frequency=data_frequency, + capital_base=capital_base, + data=data, + bundle=bundle, + bundle_timestamp=bundle_timestamp, + start=start, + end=end, + output=os.devnull, + trading_calendar=trading_calendar, + print_algo=False, + metrics_set=metrics_set, + local_namespace=False, + environ=environ, + loaders=loaders) From 9451c4ec66d82b21241cab5596adcfc05e5d5ffe Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:52:33 -0400 Subject: [PATCH 10/14] formatting --- zipline/utils/run_algo.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index fc131e281b..4f7d4addff 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -52,9 +52,7 @@ def __str__(self): return self.pyfunc_msg -def _run( - loaders, - handle_data, +def _run(handle_data, initialize, before_trading_start, analyze, @@ -73,7 +71,8 @@ def _run( print_algo, metrics_set, local_namespace, - environ): + environ, + loaders,): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. From 67ee985806b1f478d7058929997514997dc2c2a5 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:53:12 -0400 Subject: [PATCH 11/14] formatting --- zipline/utils/run_algo.py | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 4f7d4addff..1160099694 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -53,26 +53,26 @@ def __str__(self): def _run(handle_data, - initialize, - before_trading_start, - analyze, - algofile, - algotext, - defines, - data_frequency, - capital_base, - data, - bundle, - bundle_timestamp, - start, - end, - output, - trading_calendar, - print_algo, - metrics_set, - local_namespace, - environ, - loaders,): + initialize, + before_trading_start, + analyze, + algofile, + algotext, + defines, + data_frequency, + capital_base, + data, + bundle, + bundle_timestamp, + start, + end, + output, + trading_calendar, + print_algo, + metrics_set, + local_namespace, + environ, + loaders): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. From 01e3ef9912c2cbe6c4095ba8850ffa7656b34eac Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:56:30 -0400 Subject: [PATCH 12/14] sorry for all these formatting changes --- zipline/utils/run_algo.py | 43 ++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 1160099694..44fcc91a00 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -383,24 +383,25 @@ def run_algorithm(start, 'cannot specify `bundle_timestamp` without passing `bundle`', ) - return _run(handle_data=handle_data, - initialize=initialize, - before_trading_start=before_trading_start, - analyze=analyze, - algofile=None, - algotext=None, - defines=(), - data_frequency=data_frequency, - capital_base=capital_base, - data=data, - bundle=bundle, - bundle_timestamp=bundle_timestamp, - start=start, - end=end, - output=os.devnull, - trading_calendar=trading_calendar, - print_algo=False, - metrics_set=metrics_set, - local_namespace=False, - environ=environ, - loaders=loaders) + return _run( + handle_data=handle_data, + initialize=initialize, + before_trading_start=before_trading_start, + analyze=analyze, + algofile=None, + algotext=None, + defines=(), + data_frequency=data_frequency, + capital_base=capital_base, + data=data, + bundle=bundle, + bundle_timestamp=bundle_timestamp, + start=start, + end=end, + output=os.devnull, + trading_calendar=trading_calendar, + print_algo=False, + metrics_set=metrics_set, + local_namespace=False, + environ=environ, + loaders=loaders) From 339923bd90640384e3400449f9e432788b4d632e Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 12:57:06 -0400 Subject: [PATCH 13/14] just trying to keep consistent with what was already here --- zipline/utils/run_algo.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 44fcc91a00..232c0ca5b3 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -404,4 +404,5 @@ def run_algorithm(start, metrics_set=metrics_set, local_namespace=False, environ=environ, - loaders=loaders) + loaders=loaders + ) From ebcf74749ea8a26194bbe1fe7e984d6cafe20189 Mon Sep 17 00:00:00 2001 From: Cal Mitchell Date: Thu, 31 May 2018 15:11:57 -0400 Subject: [PATCH 14/14] changed language to show that this argument is for loading dataframes --- zipline/utils/run_algo.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/zipline/utils/run_algo.py b/zipline/utils/run_algo.py index 232c0ca5b3..533ca88f4e 100644 --- a/zipline/utils/run_algo.py +++ b/zipline/utils/run_algo.py @@ -72,7 +72,7 @@ def _run(handle_data, metrics_set, local_namespace, environ, - loaders): + data_frame_loaders): """Run a backtest for the given algorithm. This is shared between the cli and :func:`zipline.run_algo`. @@ -169,8 +169,8 @@ def _run(handle_data, def choose_loader(column): if column in USEquityPricing.columns: return pipeline_loader - elif column in loaders: - return loaders[column] + elif data_frame_loaders and column in data_frame_loaders: + return data_frame_loaders[column] else: raise ValueError( "No PipelineLoader registered for column %s." % column @@ -290,7 +290,7 @@ def run_algorithm(start, extensions=(), strict_extensions=True, environ=os.environ, - loaders=None): + data_frame_loaders=None): """Run a trading algorithm. Parameters @@ -404,5 +404,5 @@ def run_algorithm(start, metrics_set=metrics_set, local_namespace=False, environ=environ, - loaders=loaders + data_frame_loaders=data_frame_loaders )