diff --git a/Configuration/ToolboxArgumentParser.cs b/Configuration/ToolboxArgumentParser.cs index 4da42c213655..805c750e1913 100644 --- a/Configuration/ToolboxArgumentParser.cs +++ b/Configuration/ToolboxArgumentParser.cs @@ -38,7 +38,6 @@ public static class ToolboxArgumentParser + "/DukascopyDownloader" + "/FxcmDownloader or FDL/FxcmVolumeDownload or FVDL/GoogleDownloader or GDL/IBDownloader or IBDL" + "/OandaDownloader or ODL" - + "/AlgoSeekFuturesConverter or ASFC" + "/IVolatilityEquityConverter or IVEC/KaikoDataConverter or KDC/NseMarketDataConverter or NMDC" + "/CoarseUniverseGenerator or CUG/\n" + "RandomDataGenerator or RDG\n" @@ -55,7 +54,7 @@ public static class ToolboxArgumentParser new CommandLineOption("to-date", CommandOptionType.SingleValue, "[OPTIONAL for downloaders] If not provided 'DateTime.UtcNow' will " + "be used. --to-date=yyyyMMdd-HH:mm:ss"), new CommandLineOption("exchange", CommandOptionType.SingleValue, "[REQUIRED for CryptoiqDownloader] [Optional for KaikoDataConverter] The exchange to process, if not defined, all exchanges will be processed."), - new CommandLineOption("date", CommandOptionType.SingleValue, "[REQUIRED for AlgoSeekFuturesConverter, AlgoSeekOptionsConverter, KaikoDataConverter]" + new CommandLineOption("date", CommandOptionType.SingleValue, "[REQUIRED for KaikoDataConverter]" + "Date for the option bz files: --date=yyyyMMdd"), new CommandLineOption("source-dir", CommandOptionType.SingleValue, "[REQUIRED for IVolatilityEquityConverter, KaikoDataConverter," + " NseMarketDataConverter]"), diff --git a/ToolBox/AlgoSeekFuturesConverter/AlgoSeek.US.Futures.PriceMultipliers.1.1.csv b/ToolBox/AlgoSeekFuturesConverter/AlgoSeek.US.Futures.PriceMultipliers.1.1.csv deleted file mode 100644 index a3e0cca6a591..000000000000 --- a/ToolBox/AlgoSeekFuturesConverter/AlgoSeek.US.Futures.PriceMultipliers.1.1.csv +++ /dev/null @@ -1,262 +0,0 @@ -Globex,Product Name,MultipleFactor,Info -10Y,Micro 10-Year Yield Futures,0.1, -1S,Propane Non-LDH Mont Belvieu (OPIS) BALMO Futures,1, -22,Argus Propane Far East Index BALMO Futures,1, -2YY,Micro 2-Year Yield Futures,0.1, -30Y,Micro 30-Year Yield Futures,0.1, -5YY,Micro 5-Year Yield Futures,0.1 -6A,Australian Dollar Futures,0.01, -6B,British Pound Futures,0.01, -6C,Canadian Dollar Futures,0.01, -6E,Euro FX Futures,0.01, -6J,Japanese Yen Futures,0.0001, -6L,Brazilian Real Futures,0.01, -6M,Mexican Peso Futures,0.0001, -6N,New Zealand Dollar Futures,0.01, -6R,Russian Ruble Futures,0.0001, -6S,Swiss Franc Futures,0.01, -6Z,South African Rand Futures,0.0001, -A0D,Mini European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1, -A0F,Mini Singapore Fuel Oil 180 cst (Platts) Futures,0.1, -A1L,Gulf Coast ULSD (Platts) Up-Down BALMO Futures,1, -A1M,Gulf Coast Jet (Platts) Up-Down BALMO Futures,1, -A1R,Propane Non-LDH Mont Belvieu (OPIS) Futures,0.001, -A32,European Propane CIF ARA (Argus) BALMO Futures,1, -A3G,Premium Unleaded Gasoline 10 ppm FOB MED (Platts) Futures,0.1, -A7E,Argus Propane Far East Index Futures,0.1, -A7I,Gasoline Euro-bob Oxy NWE Barges (Argus) Crack Spread BALMO Futures,1, -A7Q,Mont Belvieu Natural Gasoline (OPIS) Futures,0.001, -A8J,Mont Belvieu Normal Butane (OPIS) BALMO Futures,1, -A8K,Conway Propane (OPIS) Futures,0.001, -A8O,Mont Belvieu LDH Propane (OPIS) BALMO Futures,1, -A91,Argus Propane Far East Index vs. European Propane CIF ARA (Argus) Futures,0.1, -A9N,Argus Propane (Saudi Aramco) Futures,0.1, -AA6,Group Three ULSD (Platts) vs. NY Harbor ULSD Futures,0.01, -AA8,Group Three Sub-octane Gasoline (Platts) vs. RBOB Futures,0.01, -ABS,Singapore Fuel Oil 180 cst (Platts) BALMO Futures,1, -ABT,Singapore Fuel Oil 380 cst (Platts) BALMO Futures,1, -AC0,Mont Belvieu Ethane (OPIS) Futures,0.001, -ACD,Australian Dollar/Canadian Dollar Futures,0.01, -AD0,Mont Belvieu Normal Butane (OPIS) Futures,0.001, -ADB,Brent Crude Oil vs. Dubai Crude Oil (Platts) Futures,0.1, -AE5,Argus LLS vs. WTI (Argus) Trade Month Futures,1, -AGA,Singapore Gasoil (Platts) vs. Low Sulphur Gasoil Futures,0.1, -AJL,Los Angeles CARBOB Gasoline (OPIS) vs. RBOB Gasoline Futures,1, -AJS,Los Angeles Jet (OPIS) vs. NY Harbor ULSD Futures,0.01, -AJY,Australian Dollar/Japanese Yen Futures,1, -AKL,Los Angeles CARB Diesel (OPIS) vs. NY Harbor ULSD Futures,0.01, -AKZ,European Naphtha (Platts) BALMO Futures,1, -ANE,Australian Dollar/New Zealand Dollar Futures,0.01, -APS,European Propane CIF ARA (Argus) Futures,1, -AR0,Mont Belvieu Natural Gasoline (OPIS) BALMO Futures,1, -ARE,RBOB Gasoline Crack Spread Futures,1, -AUP,Aluminum MW U.S. Transaction Premium Platts (25MT) Futures,0.001, -AVZ,Gulf Coast HSFO (Platts) BALMO Futures,1, -AW,Bloomberg Commodity Index Futures,10, -AYV,Mars (Argus) vs. WTI Trade Month Futures,1, -AYX,Mars (Argus) vs. WTI Financial Futures,1, -AZ1,Ethanol T2 FOB Rdam Including Duty (Platts) Futures,0.1, -B0,Mont Belvieu LDH Propane (OPIS) Futures,0.001, -B7H,Gasoline Euro-bob Oxy NWE Barges (Argus) Futures,0.1, -BCF,Black Sea Corn Financially Settled (Platts) Futures,1, -BIO,E-mini Nasdaq-100 Biotechnology Index Futures,1, -BK,WTI-Brent Financial Futures,1, -BOO,3.5% Fuel Oil Barges FOB Rdam (Platts) Crack Spread (1000mt) Futures,0.1, -BR7,Gasoline Euro-bob Oxy NWE Barges (Argus) BALMO Futures,1, -BTC,Bitcoin Futures,100, -BWF,Black Sea Wheat Financially Settled (Platts) Futures,1, -BZ,Brent Last Day Financial Futures,1, -CB,Cash-settled Butter Futures,0.001, -CJY,Canadian Dollar/Japanese Yen Futures,1, -CL,Crude Oil Futures,1, -CNH,Standard-Size USD/Offshore RMB (CNH) Futures,0.01, -CRB,Gulf Coast CBOB Gasoline A2 (Platts) vs. RBOB Gasoline Futures,0.01, -CSC,Cash-Settled Cheese Futures,0.1, -CSW,Clearbrook Bakken Sweet Crude Oil Monthly Index (Net Energy) Futures,0.1, -CSX,WTI Financial Futures,1, -CU,Chicago Ethanol (Platts) Futures,0.001, -D1N,Singapore Mogas 92 Unleaded (Platts) Brent Crack Spread Futures,0.1, -DC,Class III Milk Futures,1, -DCB,Dubai Crude Oil (Platts) Financial Futures,0.1, -DY,Dry Whey Futures,0.001, -E6,Japan C&F Naphtha (Platts) BALMO Futures,1, -E7,E-mini Euro FX Futures,0.001, -EAD,Euro/Australian Dollar Futures,0.01, -ECD,Euro/Canadian Dollar Futures,0.01, -EDP,Aluminium European Premium Duty-Paid (Metal Bulletin) Futures,1, -EH,Ethanol Futures,0.1, -EI,E-mini FTSE Emerging Index Futures,1, -EMD,E-mini S&P MidCap 400 Futures,1, -EN,European Naphtha (Platts) Crack Spread Futures,0.001, -EPN,European Propane CIF ARA (Argus) vs. Naphtha Cargoes CIF NWE (Platts) Futures,0.1, -ES,E-mini S&P 500 Futures,1, -ESK,Euro/Swedish Krona Futures,0.01, -ETH,Ether Futures,1, -EVC,Singapore Fuel Oil 380 cst (Platts) vs. European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,1, -EWG,East-West Gasoline Spread (Platts-Argus) Futures,0.1, -EWN,East-West Naphtha: Japan C&F vs. Cargoes CIF NWE Spread (Platts) Futures,0.1, -EXR,"RBOB Gasoline vs. Euro-bob Oxy NWE Barges (Argus) (350,000 gallons) Futures",0.01, -F1U,5-Year USD MAC Swap Futures,100, -FO,3.5% Fuel Oil Barges FOB Rdam (Platts) Crack Spread Futures,0.1, -FRC,Freight Route TC14 (Baltic) Futures,1, -FSS,1% Fuel Oil Cargoes FOB NWE (Platts) vs. 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1, -GC,Gold Futures,10, -GCU,Gulf Coast HSFO (Platts) vs. European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1, -GD,S&P-GSCI Commodity Index Futures,1, -GDK,Class IV Milk Futures,1, -GE,Eurodollar Futures,1, -GF,Feeder Cattle Futures,0.001, -GNF,Non-fat Dry Milk Futures,0.001, -HCL,WTI Houston Crude Oil Futures,1, -HE,Lean Hog Futures,0.001, -HG,Copper Futures,0.01, -HH,Natural Gas (Henry Hub) Last-day Financial Futures,0.1, -HO,NY Harbor ULSD Futures,0.01, -HP,Natural Gas (Henry Hub) Penultimate Financial Futures,0.1, -HRC,U.S. Midwest Domestic Hot-Rolled Coil Steel (CRU) Index Futures,1, -HTT,WTI Houston (Argus) vs. WTI Trade Month Futures,1, -IBV,USD-Denominated Ibovespa Index Futures,100, -J7,E-mini Japanese Yen Futures,0.00001, -JA,Japan C&F Naphtha (Platts) Futures,0.1, -JET,NY Buckeye Jet Fuel (Platts) vs. NY Harbor ULSD Futures,0.01, -JTB,NY Buckeye Jet Fuel (Platts) vs. NY Harbor ULSD BALMO Futures,1, -KE,KC HRW Wheat Futures,1, -KRW,Korean Won Futures,0.00001, -LBR,Lumber Futures,1, -LBS,Random Length Lumber Futures,1, -LE,Live Cattle Futures,0.001, -LIB,7-year Eris Swap Futures,1, -LID,4-year Eris Swap Futures,1, -LIE,30-year Eris Swap Futures,1, -LIT,2-Year Eris Swap Futures,1, -LIW,5-year Eris Swap Futures,1, -LIY,10-year Eris Swap Futures,1, -LT,Gulf Coast ULSD (Platts) Up-Down Futures,0.01, -M1B,Micro Gasoil 0.1% Barges FOB ARA (Platts) Futures,1, -M2K,Micro E-mini Russell 2000 Index Futures,1, -M35,Micro European 3.5% Fuel Oil Cargoes FOB Med (Platts) Futures,0.1, -M5F,Micro Coal (API 5) fob Newcastle (Argus/McCloskey) Futures,1, -M6A,E-micro Australian Dollar/American Dollar Futures,100, -M6B,E-micro British Pound/American Dollar Futures,100, -M6C,Micro USD/CAD Futures,100, -M6E,E-micro Euro/American Dollar Futures,100, -M6J,Micro USD/JPY Futures,100, -M6S,Micro USD/CHF Futures,1000, -MAE,Mini Argus Propane Far East Index Futures,0.1, -MAF,Micro Singapore Fuel Oil 380CST (Platts) Futures,1, -MBT,Micro Bitcoin Futures,100, -MCD,E-micro Canadian Dollar/American Dollar Futures,100, -MCL,Micro WTI Crude Oil Futures,1, -ME,Gulf Coast Jet (Platts) Up-Down Futures,0.01, -MEF,Micro European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1, -MEE,Mini European Naphtha (Platts) BALMO Futures,1, -MEO,Mini Gasoline Euro-bob Oxy NWE Barges (Argus) Futures,0.1, -MES,Micro E-mini Standard and Poor's 500 Stock Price Index Futures,1, -MET,Micro Ether Futures,1, -MFB,Gulf Coast HSFO (Platts) Futures,1, -MFF,Coal (API4) FOB Richards Bay (ARGUS-McCloskey) Futures,1, -MGB,Mini Gasoil 0.1 Barges FOB Rdam (Platts) vs. Low Sulphur Gasoil Futures,0.1, -MGC,E-micro Gold Futures,10, -MGT,Micro Gold TAS,1, -MIR,E-micro Indian Rupee/USD Futures,1, -MJN,Mini Japan C&F Naphtha (Platts) Futures,0.1, -MJY,E-micro Japanese Yen/American Dollar Futures,100, -MM,New York Harbor Residual Fuel 1.0% (Platts) Futures,1, -MMF,Mini 3.5% Fuel Oil Cargoes FOB MED (Platts) Financial Futures,0.1, -MNC,Mini European Naphtha CIF NWE (Platts) Futures,0.1, -MNH,Micro USD/CNH Futures,100, -MNQ,Micro E-mini Nasdaq-100 Index Futures,1, -MPS,Mini European Propane CIF ARA (Argus) Futures,2, -MSF,E-micro Swiss Franc/American Dollar Futures,100, -MSG,Mini Singapore Gasoil (Platts) Futures,0.1, -MTB,Mini Singapore Fuel Oil 380 cst (Platts) BALMO Futures,3, -MTF,Coal (API2) CIF ARA (ARGUS-McCloskey) Futures,1, -MTS,Mini Singapore Fuel Oil 380 cst (Platts) Futures,0.1, -MYM,Micro E-mini Dow Jones Industrial Average Index Futures,100, -N1B,Singapore Mogas 92 Unleaded (Platts) Futures,4, -N1U,10-Year USD MAC Swap Futures,5, -NBB,Naphtha Cargoes CIF NWE (Platts) Crack Spread (1000mt) BALMO Futures,6, -NG,Henry Hub Natural Gas Futures,0.1, -NIY,Nikkei/Yen Futures,100, -NKD,Nikkei/USD Futures,100, -NN,Henry Hub Natural Gas Last Day Financial Futures,0.1, -NOK,Norwegian Krone Futures,0.001, -NOO,Naphtha Cargoes CIF NWE (Platts) Crack Spread (1000mt) Futures,0.01, -NQ,E-mini Nasdaq-100 Futures,1, -PA,Palladium Futures,1, -PAM,Micro Palladium Futures,1, -PJY,British Pound/Japanese Yen Futures,1, -PL,Platinum Futures,10, -PLN,Polish Zloty Futures,0.001, -PSF,British Pound/Swiss Franc Futures,0.01, -QC,E-mini Copper Futures,0.01, -QG,E-mini Natural Gas Futures,0.1, -QI,E-mini Silver Futures,0.01, -QM,E-mini Crude Oil Futures,0.1, -QO,E-mini Gold Futures,1, -R5O,Micro European FOB Rdam Marine Fuel 0.5% Barges (Platts) Futures,1, -RB,RBOB Gasoline Futures,0.01, -RBB,RBOB Gasoline Brent Crack Spread Futures,0.1, -RF,Euro/Swiss Franc Futures,0.01, -RP,Euro/British Pound Futures,0.001, -RS1,E-mini Russell 1000 Index Futures,1, -RSG,E-mini Russell 1000 Growth Index Futures,1, -RSV,E-mini Russell 1000 Value Index Futures,1, -RTY,E-mini Russell 2000 Index Futures,1, -RVR,Gulf Coast Unl 87 Gasoline M2 (Platts) vs. RBOB Gasoline Futures,0.01, -RX,Dow Jones Real Estate Futures,10, -RY,Euro/Japanese Yen Futures,1, -SDA,S&P 500 Annual Dividend Index Futures,0.1, -SE,Singapore Fuel Oil 380 cst (Platts) Futures,0.1, -SEK,Swedish Krona Futures,0.001, -S5O,Micro Singapore FOB Marine Fuel 0.5% (Platts) Futures,1, -SI,Silver Futures,0.1, -SIL,1000-oz. Silver Futures,0.1, -SIR,Indian Rupee/USD Futures,1, -SON,Quarterly IMM SONIA Futures,8, -SP,S&P 500 Futures,1, -SR1,One-Month SOFR Futures,9, -SR3,Three-Month SOFR Futures,10, -T7K,Gasoline Euro-bob Oxy NWE Barges (Argus) Crack Spread Futures,0.1, -TIO,Iron Ore 62% Fe CFR China (TSI) Futures,1, -TL,Freight Route TD3C (Baltic) Futures,11, -TM,Freight Route TC2 (Baltic) Futures,1, -TN,Ultra 10-Year U.S. Treasury Note Futures,100, -TPY,Yen Denominated TOPIX Futures,1, -TRI,S&P 500 Total Return Index Futures,1, -UA,Singapore Fuel Oil 180 cst (Platts) Futures,0.1, -UB,Ultra U.S. Treasury Bond Futures,100, -UME,Urea (Granular) FOB Middle East Futures,1, -UN,European Naphtha Cargoes CIF NWE (Platts) Futures,0.1, -UV,European 3.5% Fuel Oil Barges FOB Rdam (Platts) Futures,0.1, -VX,Cboe Volatility Index (VIX),1, -WCW,Western Canadian Select Oil (Net Energy) Monthly Index Futures,1, -WTT,WTI Midland (Argus) vs. WTI Trade Month Futures,1, -XAB,E-mini Materials Select Sector Futures,1, -XAE,E-mini Energy Select Sector Futures,1, -XAF,E-mini Financial Select Sector Futures,1, -XAI,E-mini Industrial Select Sector Futures,1, -XAK,E-mini Technology Select Sector Futures,1, -XAP,E-mini Consumer Staples Select Sector Futures,1, -XAR,E-mini Real Estate Select Sector Futures,1, -XAU,E-mini Utilities Select Sector Futures,1, -XAV,E-mini Health Care Select Sector Futures,1, -XAY,E-mini Consumer Discretionary Select Sector Futures,1, -XAZ,E-mini Communication Services Select Sector Futures,1, -XC,Mini-Corn Futures,100, -XK,Mini Soybean Futures,100, -XW,Mini-sized Chicago SRW Wheat Futures,100, -YM,E-mini Dow ($5) Futures,100, -YO,Sugar # 11 CME Globex Futures,0.01, -ZB,U.S. Treasury Bond Futures,100, -ZC,Corn Futures,1, -ZF,5-Year T-Note Futures,100, -ZL,Soybean Oil Futures,0.01, -ZM,Soybean Meal Futures,10, -ZN,10-Year T-Note Futures,100, -ZO,Oats Futures,1, -ZQ,30 Day Federal Funds Futures,1, -ZR,Rough Rice Futures,0.1, -ZS,Soybean Futures,1, -ZT,2-Year T-Note Futures,100, -ZW,Chicago SRW Wheat Futures,1, diff --git a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesConverter.cs b/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesConverter.cs deleted file mode 100644 index c44ec9d51d4a..000000000000 --- a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesConverter.cs +++ /dev/null @@ -1,306 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.IO.Compression; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using QuantConnect.Data.Market; -using QuantConnect.Logging; - -namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter -{ - using Processors = Dictionary>>; - - /// - /// Process a directory of algoseek futures files into separate resolutions. - /// - public class AlgoSeekFuturesConverter - { - private const int ExecTimeout = 60;// sec - private readonly DirectoryInfo _source; - private readonly DirectoryInfo _remote; - private readonly string _destination; - private readonly List _resolutions; - private readonly DateTime _referenceDate; - private readonly HashSet _symbolFilter; - - /// - /// Create a new instance of the AlgoSeekFutures Converter. Parse a single input directory into an output. - /// - /// Convert this resolution - /// Datetime to be added to the milliseconds since midnight. Algoseek data is stored in channel files (XX.bz2) and in a source directory - /// Remote directory of the .bz algoseek files - /// Source directory of the .csv algoseek files - /// Destination directory of the processed future files - /// Collection of underlying ticker to process. - public AlgoSeekFuturesConverter(List resolutions, DateTime referenceDate, string remote, string source, string destination, HashSet symbolFilter = null) - { - _source = new DirectoryInfo(source); - _remote = new DirectoryInfo(remote); - _referenceDate = referenceDate; - _destination = destination; - _resolutions = resolutions; - _symbolFilter = symbolFilter; - } - - /// - /// Give the reference date and source directory, convert the algoseek data into n-resolutions LEAN format. - /// - public void Convert() - { - Log.Trace("AlgoSeekFuturesConverter.Convert(): Copying remote raw data files locally."); - //Get the list of available raw files, copy from its remote location to a local folder and then for each file open a separate streamer. - - var files = GetFilesInRawFolder() - .Where(f => (f.Extension == ".gz" || f.Extension == ".bz2") && !f.Name.Contains("option")) - .Select(remote => remote.CopyTo(Path.Combine(Path.GetTempPath(), remote.Name), true)) - .ToList(); - - Log.Trace("AlgoSeekFuturesConverter.Convert(): Loading {0} AlgoSeekFuturesReader for {1} ", files.Count(), _referenceDate); - - //Initialize parameters - var totalLinesProcessed = 0L; - var totalFiles = files.Count(); - var totalFilesProcessed = 0; - var start = DateTime.MinValue; - - var symbolMultipliers = LoadSymbolMultipliers(); - - //Extract each file massively in parallel. - Parallel.ForEach(files, file => - { - try - { - Log.Trace("Remote File :" + file); - - var csvFile = Path.Combine(_source.FullName, Path.GetFileNameWithoutExtension(file.Name)); - - Log.Trace("Source File :" + csvFile); - - if (!File.Exists(csvFile)) - { - // create the directory first or else 7z will fail - var csvFileInfo = new FileInfo(csvFile); - Directory.CreateDirectory(csvFileInfo.DirectoryName); - - Log.Trace("AlgoSeekFuturesConverter.Convert(): Extracting " + file); - - // Never time out extracting an archive; they can be pretty big - // and take a while to extract depending on the computer running this application - Compression.Extract7ZipArchive(file.FullName, _source.FullName, -1); - } - - // setting up local processors - var processors = new Processors(); - - var reader = new AlgoSeekFuturesReader(csvFile, symbolMultipliers, _symbolFilter); - if (start == DateTime.MinValue) - { - start = DateTime.Now; - } - - if (reader.Current != null) // reader contains the data - { - do - { - var tick = reader.Current as Tick; - - if (tick.Symbol.ID.Symbol == "VX" && ( - tick.BidPrice >= 998m || tick.AskPrice >= 998m)) - { - // Invalid value for VX futures. Invalid prices in raw data are 998/999 - continue; - } - //Add or create the consolidator-flush mechanism for symbol: - List> symbolProcessors; - if (!processors.TryGetValue(tick.Symbol, out symbolProcessors)) - { - symbolProcessors = new List>(3) - { - { _resolutions.Select(x => new AlgoSeekFuturesProcessor(tick.Symbol, _referenceDate, TickType.Trade, x, _destination)).ToList() }, - { _resolutions.Select(x => new AlgoSeekFuturesProcessor(tick.Symbol, _referenceDate, TickType.Quote, x, _destination)).ToList() }, - { _resolutions.Select(x => new AlgoSeekFuturesProcessor(tick.Symbol, _referenceDate, TickType.OpenInterest, x, _destination)).ToList() } - }; - - processors[tick.Symbol] = symbolProcessors; - } - - // Pass current tick into processor: enum 0 = trade; 1 = quote, 2 = oi - foreach (var processor in symbolProcessors[(int)tick.TickType]) - { - processor.Process(tick); - } - - if (Interlocked.Increment(ref totalLinesProcessed) % 1000000m == 0) - { - var pro = (double)processors.Values.SelectMany(p => p.SelectMany(x => x)).Count(); - var symbols = (double)processors.Keys.Count(); - Log.Trace("AlgoSeekFuturesConverter.Convert(): Processed {0,3}M ticks( {1}k / sec); Memory in use: {2} MB; Total progress: {3}%, Processor per symbol {4}", Math.Round(totalLinesProcessed / 1000000m, 2), Math.Round(totalLinesProcessed / 1000L / (DateTime.Now - start).TotalSeconds), Process.GetCurrentProcess().WorkingSet64 / (1024 * 1024), 100 * totalFilesProcessed / totalFiles, pro / symbols); - } - - } - while (reader.MoveNext()); - - Log.Trace("AlgoSeekFuturesConverter.Convert(): Performing final flush to disk... "); - Flush(processors, DateTime.MaxValue, true); - } - - processors = null; - GC.Collect(); - GC.WaitForPendingFinalizers(); - - Log.Trace("AlgoSeekFuturesConverter.Convert(): Finished processing file: " + file); - Interlocked.Increment(ref totalFilesProcessed); - } - catch(Exception err) - { - Log.Error("Exception caught! File: {0} Err: {1} Source {2} Stack {3}", file, err.Message, err.Source, err.StackTrace); - } - }); - - - } - - /// - /// Gets the files in raw folder. - /// - /// List of files in source folder - private IEnumerable GetFilesInRawFolder() - { - var files = new List(); - - var command = OS.IsLinux ? "ls" : "cmd.exe"; - var arguments = OS.IsWindows ? "/c dir /b /a-d" : string.Empty; - - var processStartInfo = new ProcessStartInfo(command, arguments) - { - CreateNoWindow = true, - WindowStyle = ProcessWindowStyle.Hidden, - UseShellExecute = false, - RedirectStandardOutput = true, - WorkingDirectory = _remote.FullName - }; - - using (var process = new Process()) - { - - process.StartInfo = processStartInfo; - process.Start(); - - while (!process.StandardOutput.EndOfStream) - { - var line = process.StandardOutput.ReadLine(); - if (line != null) - { - files.Add(new FileInfo(Path.Combine(_remote.FullName, line))); - } - } - process.WaitForExit(); - } - - return files; - - } - - /// - /// Private method loads symbol multipliers from algoseek csv file - /// - /// - private Dictionary LoadSymbolMultipliers() - { - const int columnUnderlying = 0; - const int columnMultipleFactor = 2; - - return File.ReadAllLines("AlgoSeekFuturesConverter/AlgoSeek.US.Futures.PriceMultipliers.1.1.csv") - .Select(line => line.ToCsvData()) - // skipping empty fields - .Where(line => !string.IsNullOrEmpty(line[columnUnderlying]) && - !string.IsNullOrEmpty(line[columnMultipleFactor])) - // skipping header - .Skip(1) - .ToDictionary(line => line[columnUnderlying], - line => line[columnMultipleFactor].ConvertInvariant()); - } - - private void Flush(Processors processors, DateTime time, bool final) - { - foreach (var symbol in processors.Keys) - { - processors[symbol].ForEach(p => p.ForEach(x => x.FlushBuffer(time, final))); - } - } - - /// - /// Compress the queue buffers directly to a zip file. Lightening fast as streaming ram-> compressed zip. - /// - public void Package(DateTime date) - { - var zipper = OS.IsWindows ? "C:/Program Files/7-Zip/7z.exe" : "7z"; - - Log.Trace("AlgoSeekFuturesConverter.Package(): Zipping all files ..."); - - var destination = Path.Combine(_destination, "future"); - Directory.CreateDirectory(destination); - var dateMask = date.ToStringInvariant(DateFormat.EightCharacter); - - var files = - Directory.EnumerateFiles(destination, dateMask + "*.csv", SearchOption.AllDirectories) - .GroupBy(x => Directory.GetParent(x).FullName) - .ToList(); - - // Zip each file massively in parallel - Parallel.ForEach(files, file => - //foreach (var file in files) - { - try - { - var outputFileName = file.Key + ".zip"; - - // Create and open a new ZIP file - var filesToCompress = Directory.GetFiles(file.Key, "*.csv", SearchOption.AllDirectories); - var zip = ZipFile.Open(outputFileName, ZipArchiveMode.Create); - - foreach (var fileToCompress in filesToCompress) - { - // Add the entry for each file - zip.CreateEntryFromFile(fileToCompress, Path.GetFileName(fileToCompress), CompressionLevel.Optimal); - } - - // Dispose of the object when we are done - zip.Dispose(); - - try - { - Directory.Delete(file.Key, true); - } - catch (Exception err) - { - Log.Error("Directory.Delete returned error: " + err.Message); - } - } - catch (Exception err) - { - Log.Error("File: {0} Err: {1} Source {2} Stack {3}", file, err.Message, err.Source, err.StackTrace); - } - }); - } - - } -} diff --git a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesProcessor.cs b/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesProcessor.cs deleted file mode 100644 index 348acd38500e..000000000000 --- a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesProcessor.cs +++ /dev/null @@ -1,259 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.IO; -using System.Linq; -using System.Threading; -using QuantConnect.Data.Consolidators; -using QuantConnect.Data.Market; -using QuantConnect.Logging; -using QuantConnect.Util; - -namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter -{ - /// - /// Processor for caching and consolidating ticks; - /// then flushing the ticks in memory to disk when triggered. - /// - public class AlgoSeekFuturesProcessor - { - static private int _curFileCount = 0; - private string _zipPath; - private string _entryPath; - private Symbol _symbol; - private TickType _tickType; - private Resolution _resolution; - private LazyStreamWriter _streamWriter; - private string _dataDirectory; - private IDataConsolidator _consolidator; - private DateTime _referenceDate; - private static string[] _windowsRestrictedNames = - { - "con", "prn", "aux", "nul" - }; - - /// - /// Zip entry name for the futures contract - /// - public string EntryPath - { - get - { - if (_entryPath == null) - { - _entryPath = SafeName(LeanData.GenerateZipEntryName(_symbol, _referenceDate, _resolution, _tickType)); - } - return _entryPath; - } - set { _entryPath = value; } - } - - /// - /// Zip file path for the futures contract collection - /// - public string ZipPath - { - get - { - if (_zipPath == null) - { - _zipPath = Path.Combine(_dataDirectory, SafeName(LeanData.GenerateRelativeZipFilePath(Safe(_symbol), _referenceDate, _resolution, _tickType).Replace(".zip", string.Empty))) + ".zip"; - } - return _zipPath; - } - set { _zipPath = value; } - } - - /// - /// Public access to the processor symbol - /// - public Symbol Symbol - { - get { return _symbol; } - } - - /// - /// Accessor for the final enumerator - /// - public Resolution Resolution - { - get { return _resolution; } - } - - /// - /// Type of this futures processor. - /// ASOP's are grouped trade type for file writing. - /// - public TickType TickType - { - get { return _tickType; } - set { _tickType = value; } - } - - /// - /// If no data has been consolidated, do not write to disk - /// - public bool ShouldWriteToDisk() - { - return _consolidator.Consolidated != null; - } - - /// - /// Create a new AlgoSeekFuturesProcessor for enquing consolidated bars and flushing them to disk - /// - /// Symbol for the processor - /// Reference date for the processor - /// TradeBar or QuoteBar to generate - /// Resolution to consolidate - /// Data directory for LEAN - public AlgoSeekFuturesProcessor(Symbol symbol, DateTime date, TickType tickType, Resolution resolution, string dataDirectory) - { - _symbol = Safe(symbol); - _tickType = tickType; - _referenceDate = date; - _resolution = resolution; - _dataDirectory = dataDirectory; - - // Setup the consolidator for the requested resolution - if (resolution == Resolution.Tick) - { - _consolidator = new IdentityDataConsolidator(); - } - else - { - switch (tickType) - { - case TickType.Trade: - _consolidator = new TickConsolidator(resolution.ToTimeSpan()); - break; - case TickType.Quote: - _consolidator = new TickQuoteBarConsolidator(resolution.ToTimeSpan()); - break; - case TickType.OpenInterest: - _consolidator = new OpenInterestConsolidator(resolution.ToTimeSpan()); - break; - } - } - - var path = ZipPath.Replace(".zip", string.Empty); - Directory.CreateDirectory(path); - - var file = Path.Combine(path, EntryPath); - - try - { - _streamWriter = new LazyStreamWriter(file); - } - catch (Exception err) - { - // we are unable to open new file - it is already opened due to bug in algoseek data - Log.Error("File: {0} Err: {1} Source: {2} Stack: {3}", file, err.Message, err.Source, err.StackTrace); - var newRandomizedName = (file + "-" + Math.Abs(file.GetHashCode()).ToStringInvariant()).Replace(".csv", string.Empty) + ".csv"; - - // we store the information under different (randomized) name - Log.Trace("Changing name from {0} to {1}", file, newRandomizedName); - _streamWriter = new LazyStreamWriter(newRandomizedName); - } - - // On consolidating the bars put the bar into a queue in memory to be written to disk later. - _consolidator.DataConsolidated += (sender, consolidated) => - { - _streamWriter.WriteLine(LeanData.GenerateLine(consolidated, SecurityType.Future, Resolution)); - }; - - Interlocked.Add(ref _curFileCount, 1); - if (_curFileCount % 1000 == 0) - { - Log.Trace("Opened more files: {0}", _curFileCount); - } - } - - /// - /// Process the tick; add to the con - /// - /// - public void Process(Tick data) - { - if (data.TickType != _tickType) - { - return; - } - - _consolidator.Update(data); - } - - /// - /// Write the in memory queues to the disk. - /// - /// Current foremost tick time - /// Indicates is this is the final push to disk at the end of the data - public void FlushBuffer(DateTime frontierTime, bool finalFlush) - { - //Force the consolidation if time has past the bar - _consolidator.Scan(frontierTime); - - // If this is the final packet dump it to the queue - if (finalFlush) - { - if (_consolidator.WorkingData != null) - { - _streamWriter.WriteLine(LeanData.GenerateLine(_consolidator.WorkingData, SecurityType.Future, Resolution)); - } - - _streamWriter.Flush(); - _streamWriter.Close(); - _streamWriter = null; - - Interlocked.Add(ref _curFileCount, -1); - if (_curFileCount % 1000 == 0) - { - Log.Trace("Closed some files: {0}", _curFileCount); - } - } - } - - /// - /// Add filtering to safe check the symbol for windows environments - /// - /// Symbol to rename if required - /// Renamed symbol for reserved names - private static Symbol Safe(Symbol symbol) - { - if (OS.IsWindows) - { - if (_windowsRestrictedNames.Contains(symbol.Value.ToLowerInvariant())) - { - symbol = Symbol.CreateFuture(SafeName(symbol.Underlying.Value), symbol.ID.Market, symbol.ID.Date); - } - } - return symbol; - } - private static string SafeName(string fileName) - { - if (OS.IsWindows) - { - foreach (var name in _windowsRestrictedNames) - { - // The 'con' restricted filename will corrupt the 'seCONed' filepath - var restrictedFilePath = Path.DirectorySeparatorChar + name; - var safeFilePath = Path.DirectorySeparatorChar + "_" + name; - fileName = fileName.Replace(restrictedFilePath, safeFilePath); - } - } - return fileName; - } - } -} \ No newline at end of file diff --git a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesProgram.cs b/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesProgram.cs deleted file mode 100644 index bb7ea6f47346..000000000000 --- a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesProgram.cs +++ /dev/null @@ -1,103 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using QuantConnect.Logging; -using System.Diagnostics; -using System.Globalization; -using QuantConnect.Configuration; -using System.Linq; -using System.IO; - -namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter -{ - /// - /// AlgoSeek Options Converter: Convert raw OPRA channel files into QuantConnect Options Data Format. - /// - public static class AlgoSeekFuturesProgram - { - public static void AlgoSeekFuturesConverter(string date) - { - // There are practical file limits we need to override for this to work. - // By default programs are only allowed 1024 files open; for futures parsing we need 100k - Environment.SetEnvironmentVariable("MONO_MANAGED_WATCHER", "disabled"); - Log.LogHandler = new CompositeLogHandler(new ILogHandler[] { new ConsoleLogHandler(), new FileLogHandler("log.txt") }); - - // Directory for the data, output and processed cache: - var remoteDirectory = Config.Get("futures-remote-directory").Replace("{0}", date); - var sourceDirectory = Config.Get("futures-source-directory").Replace("{0}", date); - var dataDirectory = Globals.DataFolder; - var resolutions = Config.Get("resolutions"); - var cleanSourceDirectory = Config.GetBool("clean-source-directory", false); - - Log.Trace("CONFIGURATION:"); - Log.Trace("Processor Count: " + Environment.ProcessorCount); - Log.Trace("Remote Directory: " + remoteDirectory); - Log.Trace("Source Directory: " + sourceDirectory); - Log.Trace("Destination Directory: " + dataDirectory); - - // Date for the option bz files. - var referenceDate = DateTime.ParseExact(date, DateFormat.EightCharacter, CultureInfo.InvariantCulture); - - Log.Trace("DateTime: " + referenceDate.Date.ToStringInvariant()); - - // checking if remote folder exists - if(!Directory.Exists(remoteDirectory)) - { - Log.Error("Remote Directory doesn't exist: " + remoteDirectory); - return; - } - - // prepare tick types - var resolutionList = new[] { Resolution.Minute }; - - if (!string.IsNullOrEmpty(resolutions)) - { - var names = resolutions.Split(new[] { ';' }); - resolutionList = - names - .Where(x => !string.IsNullOrEmpty(x)) - .Select(name => (Resolution)Enum.Parse(typeof(Resolution), name, true)).ToArray(); - } - - Log.Trace("Resolutions: " + string.Join(";", resolutionList.Select(x => x.ToString()).ToArray())); - - // Convert the date: - var timer = Stopwatch.StartNew(); - var converter = new AlgoSeekFuturesConverter(resolutionList.ToList() , referenceDate, remoteDirectory, sourceDirectory, dataDirectory); - converter.Convert(); - Log.Trace($"AlgoSeekFuturesConverter.Main(): {referenceDate.ToStringInvariant()} Conversion finished in time: {timer.Elapsed.ToStringInvariant(null)}"); - - // Compress the memory cache to zips. - timer.Restart(); - converter.Package(referenceDate); - Log.Trace($"AlgoSeekFuturesConverter.Main(): {referenceDate.ToStringInvariant()} Compression finished in time: {timer.Elapsed.ToStringInvariant(null)}"); - - if (cleanSourceDirectory) - { - Log.Trace($"AlgoSeekFuturesConverter.Main(): Cleaning source directory: {sourceDirectory}"); - - try - { - Directory.Delete(sourceDirectory, true); - } - catch(Exception err) - { - Log.Trace($"AlgoSeekFuturesConverter.Main(): Error while cleaning source directory {err.Message}"); - } - } - } - } -} diff --git a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesReader.cs b/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesReader.cs deleted file mode 100644 index a9f5d858a4c6..000000000000 --- a/ToolBox/AlgoSeekFuturesConverter/AlgoSeekFuturesReader.cs +++ /dev/null @@ -1,287 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ - -using System; -using System.Collections; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Linq; -using QuantConnect.Data.Market; -using QuantConnect.Logging; -using QuantConnect.Securities; -using QuantConnect.Securities.Future; -using QuantConnect.Util; - -namespace QuantConnect.ToolBox.AlgoSeekFuturesConverter -{ - /// - /// Enumerator for converting AlgoSeek futures files into Ticks. - /// - public class AlgoSeekFuturesReader : IEnumerator - { - private readonly Stream _stream; - private readonly StreamReader _streamReader; - private readonly HashSet _symbolFilter; - private readonly Dictionary _symbolMultipliers; - private readonly SymbolPropertiesDatabase _symbolProperties; - - private readonly int _columnTimestamp = -1; - private readonly int _columnSecID = -1; - private readonly int _columnTicker = -1; - private readonly int _columnType = -1; - private readonly int _columnSide = -1; - private readonly int _columnQuantity = -1; - private readonly int _columnPrice = -1; - private readonly int _columnsCount = -1; - - /// - /// Enumerate through the lines of the algoseek files. - /// - /// BZ File for AlgoSeek - /// Symbol price multiplier - /// Symbol filter to apply, if any - public AlgoSeekFuturesReader(string file, Dictionary symbolMultipliers, HashSet symbolFilter = null) - { - var streamProvider = StreamProvider.ForExtension(Path.GetExtension(file)); - _stream = streamProvider.Open(file).First(); - _streamReader = new StreamReader(_stream); - _symbolFilter = symbolFilter; - _symbolMultipliers = symbolMultipliers.ToDictionary(); - _symbolProperties = SymbolPropertiesDatabase.FromDataFolder(); - - // detecting column order in the file - var headerLine = _streamReader.ReadLine(); - if (!string.IsNullOrEmpty(headerLine)) - { - var header = headerLine.ToCsv(); - _columnTimestamp = header.FindIndex(x => x == "Timestamp"); - _columnTicker = header.FindIndex(x => x == "Ticker"); - _columnType = header.FindIndex(x => x == "Type"); - _columnSide = header.FindIndex(x => x == "Side"); - _columnSecID = header.FindIndex(x => x == "SecurityID"); - _columnQuantity = header.FindIndex(x => x == "Quantity"); - _columnPrice = header.FindIndex(x => x == "Price"); - - _columnsCount = new[] { _columnTimestamp, _columnTicker, _columnType, _columnSide, _columnSecID, _columnQuantity, _columnPrice }.Max(); - } - //Prime the data pump, set the current. - Current = null; - MoveNext(); - } - - /// - /// Parse the next line of the algoseek future file. - /// - /// - public bool MoveNext() - { - string line; - Tick tick = null; - while (tick == null && (line = _streamReader.ReadLine()) != null) - { - // If line is invalid continue looping to find next valid line. - tick = Parse(line); - } - - Current = tick; - return Current != null; - } - - /// - /// Current top of the tick file. - /// - public Tick Current { get; private set; } - - /// - /// Gets the current element in the collection. - /// - /// - /// The current element in the collection. - /// - object IEnumerator.Current => Current; - - /// - /// Reset the enumerator for the AlgoSeekFuturesReader - /// - public void Reset() - { - throw new NotImplementedException("Reset not implemented for AlgoSeekFuturesReader."); - } - - /// - /// Dispose of the underlying AlgoSeekFuturesReader - /// - public void Dispose() - { - _stream.Close(); - _stream.Dispose(); - _streamReader.Close(); - _streamReader.Dispose(); - } - - /// - /// Parse a string line into a future tick. - /// - /// - /// - private Tick Parse(string line) - { - try - { - const int TradeMask = 2; - const int QuoteMask = 1; - const int OpenInterestMask = 11; - const int MessageTypeMask = 15; - - // parse csv check column count - var csv = line.ToCsv(); - if (csv.Count - 1 < _columnsCount) - { - return null; - } - - var ticker = csv[_columnTicker]; - - // we filter out options and spreads - if (ticker.IndexOfAny(new [] { ' ', '-' }) != -1) - { - return null; - } - - ticker = ticker.Trim('"'); - - if (string.IsNullOrEmpty(ticker)) - { - return null; - } - - // ignoring time zones completely -- this is all in the 'data-time-zone' - var timeString = csv[_columnTimestamp]; - var time = DateTime.ParseExact(timeString, "yyyyMMddHHmmssFFF", CultureInfo.InvariantCulture); - - var symbol = SymbolRepresentation.ParseFutureSymbol(ticker, time.Year); - - if (symbol == null || !_symbolMultipliers.ContainsKey(symbol.ID.Symbol) || - _symbolFilter != null && !_symbolFilter.Contains(symbol.ID.Symbol, StringComparer.InvariantCultureIgnoreCase)) - { - return null; - } - - // detecting tick type (trade or quote) - TickType tickType; - bool isAsk = false; - - var type = csv[_columnType].ConvertInvariant(); - if ((type & MessageTypeMask) == TradeMask) - { - tickType = TickType.Trade; - } - else if ((type & MessageTypeMask) == OpenInterestMask) - { - tickType = TickType.OpenInterest; - } - else if ((type & MessageTypeMask) == QuoteMask) - { - tickType = TickType.Quote; - - switch (csv[_columnSide]) - { - case "B": - isAsk = false; - break; - case "S": - isAsk = true; - break; - default: - { - return null; - } - } - } - else - { - return null; - } - - // All futures but VIX are delivered with a scale factor of 10000000000. - var scaleFactor = symbol.ID.Symbol == "VX" ? decimal.One : 10000000000m; - - var price = csv[_columnPrice].ToDecimal() / scaleFactor; - var quantity = csv[_columnQuantity].ToInt32(); - - price *= _symbolMultipliers[symbol.ID.Symbol]; - - switch (tickType) - { - case TickType.Quote: - - var tick = new Tick - { - Symbol = symbol, - Time = time, - TickType = tickType, - Value = price - }; - - if (isAsk) - { - tick.AskPrice = price; - tick.AskSize = quantity; - } - else - { - tick.BidPrice = price; - tick.BidSize = quantity; - } - - return tick; - - case TickType.Trade: - - tick = new Tick - { - Symbol = symbol, - Time = time, - TickType = tickType, - Value = price, - Quantity = quantity - }; - return tick; - - case TickType.OpenInterest: - - tick = new Tick - { - Symbol = symbol, - Time = time, - TickType = tickType, - Exchange = symbol.ID.Market, - Value = quantity - }; - return tick; - } - - return null; - } - catch (Exception err) - { - Log.Error(err); - Log.Trace("Line: {0}", line); - return null; - } - } - } -} diff --git a/ToolBox/LazyStreamWriter.cs b/ToolBox/LazyStreamWriter.cs deleted file mode 100644 index dc4fd79ee9f5..000000000000 --- a/ToolBox/LazyStreamWriter.cs +++ /dev/null @@ -1,84 +0,0 @@ -/* - * QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. - * Lean Algorithmic Trading Engine v2.0. Copyright 2014 QuantConnect Corporation. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. -*/ -using System.IO; - -namespace QuantConnect.ToolBox -{ - /// - /// This class wraps a so that the StreamWriter is only - /// instantiated until WriteLine() is called. This ensures that the file the StreamWriter is - /// writing to is only created if something is written to it. A StreamWriter will create a empty file - /// as soon as it is instantiated. - /// - public class LazyStreamWriter - { - private StreamWriter _streamWriter; - private readonly string _path; - - /// - /// Constructor for the - /// - /// Path to the file that should be created - public LazyStreamWriter(string path) - { - _path = path; - } - - /// - /// Wraps the WriteLine method of the StreamWriter. - /// - /// The line to write - /// Will instantiate the StreamWriter if this is the first time this method is called - public void WriteLine(string line) - { - PrepareStreamWriter(); - - _streamWriter.WriteLine(line); - } - - /// - /// Wraps the method - /// - public void Flush() - { - if (_streamWriter != null) - { - _streamWriter.Flush(); - } - } - - /// - /// Wraps the method - /// - public void Close() - { - if (_streamWriter != null) - { - _streamWriter.Close(); - } - } - - /// - /// Checks if the StreamWriter is instantiated. If not, it will instantiate the StreamWriter - /// - private void PrepareStreamWriter() - { - if (_streamWriter == null) - { - _streamWriter = new StreamWriter(_path); - } - } - } -} diff --git a/ToolBox/Program.cs b/ToolBox/Program.cs index 19f8c8159cfb..f8e0eeb8d300 100644 --- a/ToolBox/Program.cs +++ b/ToolBox/Program.cs @@ -15,7 +15,6 @@ using QuantConnect.Configuration; using QuantConnect.Interfaces; using QuantConnect.Logging; -using QuantConnect.ToolBox.AlgoSeekFuturesConverter; using QuantConnect.ToolBox.CoarseUniverseGenerator; using QuantConnect.ToolBox.CryptoiqDownloader; using QuantConnect.ToolBox.DukascopyDownloader; @@ -89,10 +88,6 @@ var factorFileProvider { switch (targetApp) { - case "asfc": - case "algoseekfuturesconverter": - AlgoSeekFuturesProgram.AlgoSeekFuturesConverter(GetParameterOrExit(optionsObject, "date")); - break; case "ivec": case "ivolatilityequityconverter": IVolatilityEquityConverterProgram.IVolatilityEquityConverter(GetParameterOrExit(optionsObject, "source-dir"), diff --git a/ToolBox/QuantConnect.ToolBox.csproj b/ToolBox/QuantConnect.ToolBox.csproj index fbb74e72c7b2..468de2b40f74 100644 --- a/ToolBox/QuantConnect.ToolBox.csproj +++ b/ToolBox/QuantConnect.ToolBox.csproj @@ -51,9 +51,6 @@ True - - PreserveNewest - PreserveNewest diff --git a/ToolBox/README.md b/ToolBox/README.md index 4a9a6f8d6ef5..78feb17974bc 100644 --- a/ToolBox/README.md +++ b/ToolBox/README.md @@ -38,10 +38,6 @@ Example: --app=DukascopyDownloader --tickers=SPY,AAPL --resolution=Daily --from- #### Available Converters - **'--app='** - - AlgoSeekFuturesConverter or ASFC - - **'--date=yyyyMMdd'** reference date. - - AlgoSeekOptionsConverter or ASOC - - **'--date=yyyyMMdd'** reference date. - IVolatilityEquityConverter or IVEC - **'--source-dir='** source archived IVolatility data. - **'--source-meta-dir='** source archived IVolatility meta data.