Skip to content

Instantly share code, notes, and snippets.

@rhyolight

rhyolight/yo.py

Created April 18, 2016 17:41
Show Gist options
  • Save rhyolight/a62110eb36578abc09e6edbf4cb45568 to your computer and use it in GitHub Desktop.
Save rhyolight/a62110eb36578abc09e6edbf4cb45568 to your computer and use it in GitHub Desktop.
Download ZIP
Raw
yo.py
#!/usr/bin/env python
import copy
import numpy as np
import csv
import os
import json
import math
from pkg_resources import resource_filename
from nupic.encoders import SDRCategoryEncoder, ScalarEncoder, DateEncoder, MultiEncoder
from nupic.engine import Network
from nupic.data.file_record_stream import FileRecordStream as FRS
from nupic.algorithms.anomaly import computeRawAnomalyScore
_LEFT_INPUT_FILE_NAME = "LeftTestCategoryData.csv"
_RIGHT_INPUT_FILE_NAME = "RightTestCategoryData.csv"
_VERBOSITY = 0
_SEED = 1956
_LEFT_INPUT_FILE_PATH = os.path.join(os.path.dirname(__file__ ), _LEFT_INPUT_FILE_NAME)
_RIGHT_INPUT_FILE_PATH = os.path.join(os.path.dirname(__file__ ), _RIGHT_INPUT_FILE_NAME)
_OUTPUT_FILE_NAME = "Out_Network_Data.csv"
_NUM_RECORDS = 2000
#Config the link parameters
_LINK_TYPE = "UniformLink"
_LINK_PARAMS = ""
_RECORD_SENSOR = "sensorRegion"
#The left TM of the first level
_L1_LEFT_RECORD_SENSOR = "L1_Left_RecordSensor"
_L1_LEFT_SPATIAL_POOLER = "L1_Left_SpatialPooler_Region"
_L1_LEFT_TEMPORAL_POOLER = "L1_Left_TemporalPooler_Region"
_L1_LEFT_CLASSIFIER = "L1_Left_Classifier_Region"
#The right TM of the first level
_L1_RIGHT_RECORD_SENSOR = "L1_Right_RecordSensor"
_L1_RIGHT_SPATIAL_POOLER = "L1_Right_SpatialPooler_Region"
_L1_RIGHT_TEMPORAL_POOLER = "L1_Right_TemporalPooler_Region"
_L1_RIGHT_CLASSIFIER = "L1_Right_Classifier_Region"
#The TM of the second level
_L2_SPATIAL_POOLER = "L2_SpatialPooler_Region"
_L2_TEMPORAL_POOLER = "L2_TemporalPooler_Region"
_L2_CLASSIFIER = "L2_Classifier_Region"
# ---------------------------------------------------------------------------------
# Config fields for SDRCategoryEncoder
_WIDTH_ENCODER = 50
_COLUMNS_COUNT = 50
_ACTIVE_BITS_ENCODER = 15
_FORCED = True
_ACTIVATION_SHRESHOLD = 5
_CELL_PER_COLUMNS = 5
# ---------------------------------------------------------------------------------
# Config field for SPRegion
SP_PARAMS = {
"spVerbosity": _VERBOSITY,
"spatialImp": "cpp",
"globalInhibition": 1,
"columnCount": _COLUMNS_COUNT,
# This must be set before creating the SPRegion
"inputWidth": 0,
"numActiveColumnsPerInhArea": 5,
"seed": 1956,
"potentialPct": 0.8,
"synPermConnected": 0.1,
"synPermActiveInc": 0.0001,
"synPermInactiveDec": 0.0005,
"maxBoost": 1.0,
}
# Config field for TPRegion
TP_PARAMS = {
"verbosity": _VERBOSITY,
"columnCount": _COLUMNS_COUNT,
"cellsPerColumn": _CELL_PER_COLUMNS,
"inputWidth": _WIDTH_ENCODER,
"seed": 42,
"temporalImp": "cpp",
"newSynapseCount": 20,
"maxSynapsesPerSegment": 32,
"maxSegmentsPerCell": 10,
"initialPerm": 0.21,
"permanenceInc": 0.1,
"permanenceDec": 0.1,
"globalDecay": 0.0,
"maxAge": 0,
"minThreshold": 5,
"activationThreshold": 7,
"outputType": "activeState",
"pamLength": 3,
}
ClASSIFIER_PARAMS = { # Learning rate. Higher values make it adapt faster.
'alpha': 0.1,
# A comma separated list of the number of steps the
# classifier predicts in the future. The classifier will
# learn predictions of each order specified.
'steps': '1',
# The specific implementation of the classifier to use
# See CLAClassifierFactory#create for options
'implementation': 'py',
# Diagnostic output verbosity control;
# 0: silent; [1..6]: increasing levels of verbosity
'clVerbosity': 0
}
def createDataSource(inputFile):
return FRS(streamID=inputFile)
def createWriter():
outPath = os.path.join(os.path.dirname(__file__), _OUTPUT_PATH)
with open(outPath, "w") as outputFile:
return csv.writer(outputFile)
def testDataSetHelpers():
dataSource = createDataSource()
for _ in range(100):
print dataSource.next()[5]
def createEncoder():
# encoder = SDRCategoryEncoder(_WIDTH_ENCODER, _ACTIVE_BITS_ENCODER, forced=_FORCED)
encoder = MultiEncoder()
encoder.addMultipleEncoders({
"timestamp":{"fieldname":u"timestamp",
"type" :"DateEncoder",
"name" :u"datetime"
# "timeOfDay": (21, 9.5)
},
"sequence":{"fieldname": u"sequence",
"type" : "SDRCategoryEncoder",
"name" : "sequence",
"w" : _ACTIVE_BITS_ENCODER,
"n" : _WIDTH_ENCODER,
"forced" : True
}
# "index" : {"fieldname": u"category",
# "type" : "ScalarEncoder",
# "name" : u"index",
# "clipInput": True,
# "minval" : 0.0,
# "maxval" : 10.0,
# "w" : _ACTIVE_BITS_ENCODER,
# "n" : _WIDTH_ENCODER,
# "forced" : True
#
# }
})
return encoder
def createRecordSensor(network, name, dataSource):
regionType = "py.RecordSensor"
regionParams = json.dumps({"verbosity":_VERBOSITY})
network.addRegion(name, regionType, regionParams)
sensorRegion = network.regions[name].getSelf()
sensorRegion.encoder = createEncoder()
sensorRegion.dataSource = dataSource
print("Create the encoder successfully!")
return sensorRegion
def createSpatialPooler(network, name, inputWidth):
#Create the spatial pooler region
SP_PARAMS["inputWidth"] = inputWidth
spatialPoolerRegion = network.addRegion(name, "py.SPRegion", json.dumps(SP_PARAMS))
#Make sure the learning is enabled
spatialPoolerRegion.setParameter("learningMode", True)
spatialPoolerRegion.setParameter("anomalyMode", False)
return spatialPoolerRegion
def createTemporalPooler(network, name):
temporalPoolerRegion = network.addRegion(name, "py.TPRegion", json.dumps(TP_PARAMS))
temporalPoolerRegion.setParameter("topDownMode", True)
temporalPoolerRegion.setParameter("learningMode", True)
temporalPoolerRegion.setParameter("inferenceMode", True)
temporalPoolerRegion.setParameter("anomalyMode", True)
return temporalPoolerRegion
def createClassifier(network, name):
classifier = network.addRegion(name, "py.CLAClassifierRegion", json.dumps(ClASSIFIER_PARAMS))
classifier.setParameter("inferenceMode", True)
classifier.setParameter("learningMode", True)
return classifier
def createNetwork(leftDataSource, rightDataSource):
network = Network()
#-----------------------------------------FIRST LEVEL---------------------------------------------------
#Create the left side TM
LeftSensorRecorder = createRecordSensor(network, name=_L1_LEFT_RECORD_SENSOR, dataSource=leftDataSource)
createSpatialPooler(network, name=_L1_LEFT_SPATIAL_POOLER, inputWidth=LeftSensorRecorder.encoder.getWidth())
LeftTemporalPooler = createTemporalPooler(network, name=_L1_LEFT_TEMPORAL_POOLER)
createClassifier(network, name=_L1_LEFT_CLASSIFIER)
#Link the components
network.link(_L1_LEFT_RECORD_SENSOR, _L1_LEFT_SPATIAL_POOLER, _LINK_TYPE, _LINK_PARAMS)
network.link(_L1_LEFT_SPATIAL_POOLER, _L1_LEFT_TEMPORAL_POOLER, _LINK_TYPE, _LINK_PARAMS)
network.link(_L1_LEFT_TEMPORAL_POOLER, _L1_LEFT_CLASSIFIER, _LINK_TYPE,_LINK_PARAMS,
srcOutput="bottomUpOut", destInput="bottomUpIn")
#********************************************************************************************************
#IMPORTANT: Define the category source output and the destination category input!!!
network.link(_L1_LEFT_RECORD_SENSOR, _L1_LEFT_CLASSIFIER, _LINK_TYPE, _LINK_PARAMS,
srcOutput="sourceOut", destInput="categoryIn")
#********************************************************************************************************
#Create the right side TM
RightSensorRecorder = createRecordSensor(network, name=_L1_RIGHT_RECORD_SENSOR, dataSource=rightDataSource)
createSpatialPooler(network, name=_L1_RIGHT_SPATIAL_POOLER, inputWidth=LeftSensorRecorder.encoder.getWidth())
RightTemporalPooler = createTemporalPooler(network, name=_L1_RIGHT_TEMPORAL_POOLER)
createClassifier(network, name=_L1_RIGHT_CLASSIFIER)
#Link the components
network.link(_L1_RIGHT_RECORD_SENSOR, _L1_RIGHT_SPATIAL_POOLER, _LINK_TYPE, _LINK_PARAMS)
network.link(_L1_RIGHT_SPATIAL_POOLER, _L1_RIGHT_TEMPORAL_POOLER, _LINK_TYPE, _LINK_PARAMS)
network.link(_L1_RIGHT_TEMPORAL_POOLER, _L1_RIGHT_CLASSIFIER, _LINK_TYPE,_LINK_PARAMS)
network.link(_L1_RIGHT_RECORD_SENSOR, _L1_RIGHT_CLASSIFIER, _LINK_TYPE, _LINK_PARAMS,
srcOutput="sourceOut", destInput="categoryIn")
#
# #-----------------------------------------SECOND LEVEL---------------------------------------------------
l2inputWidth = LeftTemporalPooler.getSelf().getOutputElementCount("bottomUpOut") + \
RightTemporalPooler.getSelf().getOutputElementCount("bottomUpOut")
createSpatialPooler(network, name=_L2_SPATIAL_POOLER, inputWidth=l2inputWidth)
createTemporalPooler(network, name=_L2_TEMPORAL_POOLER)
# This classifier is not necessary
# createClassifier(network, name=_L2_CLASSIFIER)
#Link the components
network.link(_L1_LEFT_TEMPORAL_POOLER, _L2_SPATIAL_POOLER, _LINK_TYPE, _LINK_PARAMS,
srcOutput="bottomUpOut", destInput="bottomUpIn")
network.link(_L1_RIGHT_TEMPORAL_POOLER, _L2_SPATIAL_POOLER, _LINK_TYPE, _LINK_PARAMS,
srcOutput="bottomUpOut", destInput="bottomUpIn")
network.link(_L2_SPATIAL_POOLER, _L2_TEMPORAL_POOLER, _LINK_TYPE, _LINK_PARAMS)
network.link( _L2_TEMPORAL_POOLER, _L2_SPATIAL_POOLER, _LINK_TYPE, _LINK_PARAMS,
srcOutput="topDownOut", destInput="topDownIn")
# network.link(_L2_TEMPORAL_POOLER, _L2_CLASSIFIER, _LINK_TYPE, _LINK_PARAMS)
return network
def runClassifier(classifier, sensorRegion, tpRegion, recordNumber):
actualInput = float(sensorRegion.getOutputData("sourceOut")[1])
scalarEncoder = sensorRegion.getSelf().encoder.encoders[1][1]
bucketIndex = scalarEncoder.getBucketIndices(actualInput)[0]
tpOutput = tpRegion.getOutputData("bottomUpOut").nonzero()[0]
classDict = {"actValue": actualInput, "bucketIdx":bucketIndex}
results = classifier.getSelf().customCompute(recordNum=recordNumber,
patternNZ=tpOutput,
classification=classDict)
mostLikelyResult = sorted(zip(results[1], results["actualValues"]))[-1]
predictionConfidence = mostLikelyResult[0]
predictedValue = mostLikelyResult[1]
return actualInput, predictedValue, predictionConfidence
def runNetwork(network, numRecords, writer):
l1LeftSensorRegion = network.regions[_L1_LEFT_RECORD_SENSOR]
l1LeftSpRegion = network.regions[_L1_LEFT_SPATIAL_POOLER]
l1LeftTpRegion = network.regions[_L1_LEFT_TEMPORAL_POOLER]
l1LeftClassifier = network.regions[_L1_LEFT_CLASSIFIER]
l1RightSensorRegion = network.regions[_L1_RIGHT_RECORD_SENSOR]
l1RightSpRegion = network.regions[_L1_RIGHT_SPATIAL_POOLER]
l1RightTpRegion = network.regions[_L1_RIGHT_TEMPORAL_POOLER]
l1RightClassifier = network.regions[_L1_RIGHT_CLASSIFIER]
l2SpRegion = network.regions[_L2_SPATIAL_POOLER]
l2TpRegion = network.regions[_L2_TEMPORAL_POOLER]
# l2Classifier = network.regions[_L2_CLASSIFIER]
l1LeftPreviousPredictedColumns = []
l1RightPreviousPredictedColumns = []
l1LeftPreviousPrediction = None
l1RightPreviousPrediction = None
l2PreviousPrediction = None
l1LeftErrorSum = 0.0
l1RightErrorSum = 0.0
l2ErrorSum = 0.0
dataSource = createDataSource(_LEFT_INPUT_FILE_PATH)
for record in xrange(numRecords):
network.run(1)
spOutput = l1LeftSpRegion.getOutputData("bottomUpOut")
# print("The output of the left sp region: {}".format(spOutput))
print("The input is {}".format(dataSource.next()[1]))
lefTPOutput = l1LeftTpRegion.getOutputData("bottomUpOut")
# print("The output of the left tp region: {}".format(lefTPOutput.nonzero()))
rightTPOutput = l1RightTpRegion.getOutputData("bottomUpOut")
# print("The output of the right tp region: {}".format(rightTPOutput.nonzero()))
rightTPTopDownOutput = l1RightTpRegion.getOutputData("topDownOut")
# print("The top down output of the right tp region: {}".format(rightTPTopDownOutput))
l2SPInput = l2SpRegion.getInputData("bottomUpIn")
# print("The output of the l2 sp region: {}".format(l2SPInput))
l2SPTopDownInput = l2SpRegion.getInputData("topDownIn")
# print("The l2SPTopDownInput of the l2 sp region: {}".format(l2SPTopDownInput))
l2TPOutput = l2TpRegion.getOutputData("bottomUpOut")
# print("The l2TPOutput of the l2 sp region: {}".format(l2TPOutput))
l2TPTopDownOutput = l2TpRegion.getOutputData("topDownOut")
# print("The output of the l2 tp top down output: {}".format(l2TPTopDownOutput))
temporalTopDownOut = l2SpRegion.getOutputData("temporalTopDownOut")
# print("The temporalTopDownOut of the l2 tp top down output: {}".format(temporalTopDownOut))
spatialTopDownOut = l2SpRegion.getOutputData("spatialTopDownOut")
# print("The spatialTopDownOut of the l2 tp top down output: {}".format(spatialTopDownOut))
sensorOutput = l1LeftSensorRegion.getOutputData("sourceOut")
# print("The source output of the left sensor region: {}".format(sensorOutput))
categoryOut = l1LeftSensorRegion.getOutputData("categoryOut")
# print("The category output of the left sensor region: {}".format(categoryOut))
actualValue = l1LeftClassifier.getOutputData("actualValues")
categories = l1LeftClassifier.getOutputData("categoriesOut")
pro = l1LeftClassifier.getOutputData("probabilities")
inputData = l1LeftClassifier.getInputData("categoryIn")
# print("The actualValues of the left classifier region: {}".format(actualValue))
# print("The categories of the left classifier region: {}".format(categories))
print("The pro of the left classifier region: {}".format(pro))
# print("The input data of the left classifier region: {}".format(inputData))
# l1Actual, l1Prediction, l1Confidence = runClassifier(l1LeftClassifier,
# l1LeftSensorRegion,
# l1LeftTpRegion,
# record)
# print("L1 left actual value is: {}".format(l1Actual))
# print("L1 left actual value is: {}".format(l1Prediction))
# print("L1 left actual value is: {}".format(l1Confidence))
print("*"*60)
# actual, l1LeftPrediction, l1LeftConfidence = runClassifier(l1LeftClassifier,
# l1LeftSensorRegion,
# l1LeftTpRegion, record)
# if l1LeftPreviousPrediction is not None:
# l1LeftErrorSum += math.fabs(l1LeftPreviousPrediction - actual)
# l1LeftPreviousPrediction = l1LeftPrediction
# actual, l1RightPrediction, l1RightConfidence = runClassifier(l1RightClassifier,
# l1RightSensorRegion,
# l1RightTpRegion, record)
# if l1RightPreviousPrediction is not None:
# l1RightErrorSum += math.fabs(l1RightPreviousPrediction - actual)
# l1RightPreviousPrediction = l1RightPrediction
#
# actual, l2Prediction, l2Confidence = runClassifier(l2Classifier,
# l2SensorRegion,
# l2TpRegion, record)
# if l2PreviousPrediction is not None:
# l2ErrorSum += math.fabs(l2PreviousPrediction - actual)
# l2PreviousPrediction = l2Prediction
# l1LeftActiveColumns = l1LeftSpRegion.getOutputData("bottomUpOut").nonzero()[0]
# l1RightActiveColumns = l1RightSpRegion.getOutputData("bottomUpOut").nonzero()[0]
# l2ActiveColumns = l2SpRegion.getOutputData("bottomUpOut").nonzero()[0]
# l1LeftAnomalyScore = computeRawAnomalyScore(l1LeftActiveColumns,
# l1LeftPreviousPredictedColumns)
#Write record number, actualInput, and anomaly scores
# writer.writerow((record, actual, l1LeftAnomalyScore))
#Store the predicted columns for the next time step
# l1LeftPredictedColumns = l1LeftTpRegion.getOutputData("topDownOut").nonzero()[0]
# l1LeftPreviousPredictedColumns = copy.deepcopy(l1LeftPredictedColumns)
# l2PredictedColumns = l2TpRegion.getOutputData("topDownOut").nonzero()[0]
# l2previousPredictedColumns = copy.deepcopy(l2PredictedColumns)
# if numRecords > 1:
# print("L1 ave abs class. error: %f", (l1LeftErrorSum / (numRecords - 1)))
# print("L2 ave abs class. error: %f", (l2ErrorSum / (numRecords - 1)))
def runDemo():
leftDataSource = FRS(streamID=_LEFT_INPUT_FILE_PATH)
rightDataSource = FRS(streamID=_RIGHT_INPUT_FILE_PATH)
numRecords = leftDataSource.getDataRowCount()
print("*"*80)
print("The number of the records is: %d" %numRecords)
print("*"*80)
print("The fields are: {}".format(leftDataSource.getFields()))
print("*"*80)
print("Creating network")
network = createNetwork(leftDataSource, rightDataSource)
outputPath = os.path.join(os.path.dirname(__file__), _OUTPUT_FILE_NAME)
with open(outputPath, "w") as outputFile:
writer = csv.writer(outputFile)
print("Running network")
print("Writing output to: {}".format(outputPath))
runNetwork(network, numRecords, writer)
print("Hierarchy finished")
if __name__ == "__main__":
#testDataSetHelpers()
#createEncoder()
#runNetwork(createNetwork(createDataSource()), createWriter())
runDemo()
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment