NT8 Tick Data Experiments

InternalApexDataLoadingHours (do not edit or remove).xml

<?xml version="1.0" encoding="utf-8"?>
<NinjaTrader>
  <TradingHours xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
    <HolidaysSerializable />
    <PartialHolidaysSerializable />
    <Sessions>
      <Session>
        <BeginDay>Sunday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Monday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Sunday</TradingDay>
      </Session>
      <Session>
        <BeginDay>Monday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Tuesday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Monday</TradingDay>
      </Session>
      <Session>
        <BeginDay>Tuesday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Wednesday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Tuesday</TradingDay>
      </Session>
      <Session>
        <BeginDay>Wednesday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Thursday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Wednesday</TradingDay>
      </Session>
      <Session>
        <BeginDay>Thursday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Friday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Thursday</TradingDay>
      </Session>
      <Session>
        <BeginDay>Friday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Saturday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Friday</TradingDay>
      </Session>
      <Session>
        <BeginDay>Saturday</BeginDay>
        <BeginTime>0</BeginTime>
        <EndDay>Sunday</EndDay>
        <EndTime>0</EndTime>
        <TradingDay>Saturday</TradingDay>
      </Session>
    </Sessions>
    <TimeZone>UTC</TimeZone>
    <Version>3000</Version>
    <Name>InternalApexDataLoadingHours (do not edit or remove)</Name>
  </TradingHours>
</NinjaTrader>

TickExperiment02.cs

using ApexTough.Data;
using Newtonsoft.Json;
using NinjaTrader.Cbi;
using NinjaTrader.Data;
using NinjaTrader.Gui.Chart;
using System;
using System.Windows;

namespace NinjaTrader.NinjaScript.Indicators {

    public class TickExperiment02 : Indicator {

        private const string DataLoadingHoursName = "InternalApexDataLoadingHours (do not edit or remove)";
        private static readonly TimeZoneInfo tradingPlatformTimeZone = Core.Globals.GeneralOptions.TimeZoneInfo;

        DateTime desiredDataDate; // the specific date that we'll try to request ticks for the UTC session
        DateTime desiredDataMinTimeUtc; // the minimum timestamp for this specific utc session
        DateTime desiredDataMaxTimeUtc; // the maximum timestamp for this specific utc session

        BarsRequest longRequest; // requests bars for a long period of time including the specific UTC day
        BarsRequest specificRequest; // requests bars for a specific UTC day
        bool isLongRequestCompleted; // used to trigger results analysis
        bool isSpecificRequestCompleted; // used to trigger results analysis

        protected override void OnStateChange() {
            switch (State) {

                case State.SetDefaults:
                    Name = "_Tick Experiment 02";
                    IsOverlay = true;
                    IsChartOnly = true;
                    BarsRequiredToPlot = 0;
                    Calculate = Calculate.OnPriceChange;
                    DisplayInDataBox = true;
                    break;

                case State.Configure:
                    break;

                case State.DataLoaded:

                    if (Bars.Count > 0) {
                        // We want to accurately request all tick data for the exact UTC day 24 October 2017
                        desiredDataDate = new DateTime(2017, 10, 24, 0, 0, 0, DateTimeKind.Unspecified); // !!Important to be Unspecified
                        desiredDataMinTimeUtc = desiredDataDate.AddTicks(1); // First tick timestamp will be >= this value (one tick after midnight)
                        desiredDataMaxTimeUtc = desiredDataDate.AddDays(1); // Last tick timestamp will be <= this value (exactly midnight)

                        // Create a request that I know from experience will definitely include all the desired data.
                        longRequest = CreateTicksRequest(desiredDataDate.AddDays(-10), desiredDataDate.AddDays(10));

                        // Create a request that should include exactly the desired data.
                        // Note that the session date used for the request needs to be incremented if it's in a timezone ahead of UTC.
                        var offset = tradingPlatformTimeZone.GetUtcOffset(desiredDataDate); // !!Important that desiredDataDate.Kind == DateTimeKind.Unspecified for this to work properly
                        var specificRequestSessionDate = offset > TimeSpan.Zero ? desiredDataDate.AddDays(1) : desiredDataDate;
                        specificRequest = CreateTicksRequest(specificRequestSessionDate, specificRequestSessionDate);

                        longRequest.Request(RequestCallback);
                        specificRequest.Request(RequestCallback);
                    }
                    break;

                case State.Terminated:
                    break;
            }
        }

        BarsRequest CreateTicksRequest(DateTime from, DateTime to) {
            return new BarsRequest(Instrument, from, to) {
                IsSplitAdjusted = true,
                IsDividendAdjusted = true,
                IsResetOnNewTradingDay = true,
                MergePolicy = MergePolicy.MergeBackAdjusted,
                TradingHours = TradingHours.Get(DataLoadingHoursName), // UTC Default 24/7 template
                LookupPolicy = LookupPolicies.Provider | LookupPolicies.Repository,
                BarsPeriod = new BarsPeriod { BarsPeriodType = BarsPeriodType.Tick, Value = 1, },
            };
        }

        void RequestCallback(BarsRequest request, ErrorCode error, string message) {
            // Just trigger off the AnalyseData method after both requests have completed.
            if (error == ErrorCode.NoError) {
                lock (this) {
                    if (request == longRequest) {
                        isLongRequestCompleted = true;
                    } else if (request == specificRequest) {
                        isSpecificRequestCompleted = true;
                    }
                    if (isLongRequestCompleted && isSpecificRequestCompleted) {
                        AnalyseData();
                    }
                }
            }
        }

        void AnalyseData() {

            var results = new Results();

            // Extract the expected first and last timestamps for the desired date from the "longRequest"
            for (var i = 0; i < longRequest.Bars.Count; i++) {
                var timeUTC = TradingPlatformTimeToUTC(longRequest.Bars.GetTime(i));
                if (timeUTC >= desiredDataMinTimeUtc && timeUTC <= desiredDataMaxTimeUtc) {
                    if (results.ExpectedFirstTickTimeUTC == DateTime.MinValue) {
                        results.ExpectedFirstTickTimeUTC = timeUTC;
                    }
                    results.ExpectedTickCount++;
                    results.ExpectedLastTickTimeUTC = timeUTC;
                }
            }

            // Now find out the actual first and last timestamps achieved by the "specificRequest"
            results.ActualFirstTickTimeUTC = TradingPlatformTimeToUTC(specificRequest.Bars.GetTime(0));
            results.ActualLastTickTimeUTC = TradingPlatformTimeToUTC(specificRequest.Bars.GetTime(specificRequest.Bars.Count - 1));
            results.ActualTickCount = specificRequest.Bars.Count;

            longRequest.Dispose();
            specificRequest.Dispose();

            MessageBox.Show(JsonConvert.SerializeObject(results, Formatting.Indented));
        }

        /// <summary>
        /// Converts the trading platform timezone time to a utc time
        /// </summary>
        DateTime TradingPlatformTimeToUTC(DateTime at) {
            return new DateTime(at.Ticks - tradingPlatformTimeZone.GetUtcOffset(at).Ticks, DateTimeKind.Utc);
        }

        protected override void OnRender(ChartControl chartControl, ChartScale chartScale) {
            base.OnRender(chartControl, chartScale);
            // TODO: Add something friendly to indicate progress
        }

        class Results {
            public DateTime ExpectedFirstTickTimeUTC;
            public DateTime ExpectedLastTickTimeUTC;
            public int ExpectedTickCount;
            public DateTime ActualFirstTickTimeUTC;
            public DateTime ActualLastTickTimeUTC;
            public int ActualTickCount;
            public bool Success => ActualFirstTickTimeUTC == ExpectedFirstTickTimeUTC && ActualLastTickTimeUTC == ExpectedLastTickTimeUTC && ActualTickCount == ExpectedTickCount;
        }
    }
}

TickExperiment03.cs

using ApexTough.Data;
using Newtonsoft.Json;
using NinjaTrader.Cbi;
using NinjaTrader.Data;
using NinjaTrader.Gui.Chart;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Windows;

namespace NinjaTrader.NinjaScript.Indicators {

    public class TickExperiment03 : Indicator {

        private const string DataLoadingHoursName = "InternalApexDataLoadingHours (do not edit or remove)";
        private static readonly TimeZoneInfo tradingPlatformTimeZone = Core.Globals.GeneralOptions.TimeZoneInfo;

        // In this experiment, we'll be testing the loading of several contiguous days of
        // tick data using a number of requests for specific days.
        DateTime desiredDataStartDate; // the first specific date that we'll try to request ticks in UTC sessions
        DateTime desiredDataEndDate; // the last specific date that we'll try to request ticks in UTC sessions
        DateTime desiredDataMinTimeUtc; // the minimum timestamp for all the specific utc sessions
        DateTime desiredDataMaxTimeUtc; // the maximum timestamp for all the specific utc sessions

        BarsRequest longRequest; // requests tick bars for a long period of time including all the specific UTC days
        bool isLongRequestCompleted; // used to trigger results analysis
        List<BarsRequest> individualRequests = new List<BarsRequest>(); // all the tick requests for each specific UTC day
        List<BarsRequest> individualRequestsInProgress = new List<BarsRequest>(); // emptying this list helps trigger results analysis

        protected override void OnStateChange() {
            switch (State) {

                case State.SetDefaults:
                    Name = "_Tick Experiment 03";
                    IsOverlay = true;
                    IsChartOnly = true;
                    BarsRequiredToPlot = 0;
                    Calculate = Calculate.OnPriceChange;
                    DisplayInDataBox = true;
                    break;

                case State.Configure:
                    break;

                case State.DataLoaded:
                    if (Bars.Count > 0) {

                        // We want to accurately request all tick data for the exact UTC period 23 July 2017 until and including 23 October 2017
                        desiredDataStartDate = new DateTime(2017, 7, 23, 0, 0, 0, DateTimeKind.Unspecified); // !!Important to be Unspecified
                        desiredDataEndDate = new DateTime(2017, 10, 23, 0, 0, 0, DateTimeKind.Unspecified); // !!Important to be Unspecified
                        desiredDataMinTimeUtc = desiredDataStartDate.AddTicks(1); // First tick timestamp will be >= this value (one tick after midnight)
                        desiredDataMaxTimeUtc = desiredDataEndDate.AddDays(1); // Last tick timestamp will be <= this value (exactly midnight)

                        // Create a request that I know from experience will definitely include all the desired data.
                        longRequest = CreateRequest(desiredDataStartDate.AddDays(-10), desiredDataEndDate.AddDays(10));
                        longRequest.Request(RequestCallback);

                        // Create all the individual requests
                        lock (this) {
                            for (var date = desiredDataStartDate; date <= desiredDataEndDate; date = date.AddDays(1)) {
                                // Create a request that SHOULD include exactly the desired data.
                                // Note that the session date used for the request needs to be incremented if it's in a timezone ahead of UTC.
                                var offset = tradingPlatformTimeZone.GetUtcOffset(date); // !!Important that desiredDataDate.Kind == DateTimeKind.Unspecified for this to work properly
                                var individualRequestSessionDate = offset > TimeSpan.Zero ? date.AddDays(1) : date;
                                var individualRequest = CreateRequest(individualRequestSessionDate, individualRequestSessionDate);
                                individualRequest.Request(RequestCallback);
                                individualRequests.Add(individualRequest);
                                individualRequestsInProgress.Add(individualRequest);
                            }
                        }
                    }
                    break;

                case State.Terminated:
                    break;
            }
        }

        BarsRequest CreateRequest(DateTime from, DateTime to) {
            return new BarsRequest(Instrument, from, to) {
                IsSplitAdjusted = true,
                IsDividendAdjusted = true,
                IsResetOnNewTradingDay = true,
                MergePolicy = MergePolicy.MergeBackAdjusted,
                TradingHours = TradingHours.Get(DataLoadingHoursName), // UTC Default 24/7 template
                LookupPolicy = LookupPolicies.Provider | LookupPolicies.Repository,
                BarsPeriod = new BarsPeriod { BarsPeriodType = BarsPeriodType.Tick, Value = 1, },
            };
        }

        void RequestCallback(BarsRequest request, ErrorCode error, string message) {
            // Just trigger off the AnalyseData method after all requests have completed.
            if (error == ErrorCode.NoError) {
                lock (this) {
                    if (request == longRequest) {
                        isLongRequestCompleted = true;
                    } else {
                        individualRequestsInProgress.Remove(request);
                    }
                    if (isLongRequestCompleted && individualRequestsInProgress.Count == 0) {
                        AnalyseData();
                    }
                }
            } else {
                Debugger.Break(); // we never get here in normal circumstances, but this alerts user if for some reason we do
            }
        }

        void AnalyseData() {

            var results = new Results();

            // Extract the expected first and last timestamps for the desired date range from the "longRequest"
            for (var i = 0; i < longRequest.Bars.Count; i++) {
                var timeUTC = TradingPlatformTimeToUTC(longRequest.Bars.GetTime(i));
                if (timeUTC >= desiredDataMinTimeUtc && timeUTC <= desiredDataMaxTimeUtc) {
                    if (results.ExpectedFirstTickTimeUTC == DateTime.MinValue) {
                        results.ExpectedFirstTickTimeUTC = timeUTC;
                    }
                    results.ExpectedLastTickTimeUTC = timeUTC;
                    results.ExpectedTickCount++;
                }
            }

            // Now find out the actual first and last timestamps achieved by the "specificRequest"
            var firstSpecificRequest = individualRequests[0];
            var lastSpecificRequest = individualRequests[individualRequests.Count - 1];
            results.ActualFirstTickTimeUTC = TradingPlatformTimeToUTC(firstSpecificRequest.Bars.GetTime(0));
            results.ActualLastTickTimeUTC = TradingPlatformTimeToUTC(lastSpecificRequest.Bars.GetTime(lastSpecificRequest.Bars.Count - 1));
            results.ActualTickCount = individualRequests.Sum(r => r.Bars.Count);

            longRequest.Dispose();
            individualRequests.ForEach(r => r.Dispose());

            MessageBox.Show(JsonConvert.SerializeObject(results, Formatting.Indented));
        }

        /// <summary>
        /// Converts the trading platform timezone time to a utc time
        /// </summary>
        DateTime TradingPlatformTimeToUTC(DateTime at) {
            return new DateTime(at.Ticks - tradingPlatformTimeZone.GetUtcOffset(at).Ticks, DateTimeKind.Utc);
        }

        protected override void OnRender(ChartControl chartControl, ChartScale chartScale) {
            base.OnRender(chartControl, chartScale);
            // TODO: Add some friendly progress update text
        }

        class Results {
            public DateTime ExpectedFirstTickTimeUTC;
            public DateTime ExpectedLastTickTimeUTC;
            public int ExpectedTickCount;
            public DateTime ActualFirstTickTimeUTC;
            public DateTime ActualLastTickTimeUTC;
            public int ActualTickCount;
            public bool Success => ActualFirstTickTimeUTC == ExpectedFirstTickTimeUTC && ActualLastTickTimeUTC == ExpectedLastTickTimeUTC && ActualTickCount == ExpectedTickCount;
        }
    }
}