NPCSenses.lua

local ReplicatedStorage = game:GetService("ReplicatedStorage")
local ServerStorage = game:GetService("ServerStorage")
local CollectionService = game:GetService("CollectionService")

local ClassService = require(ServerStorage.Services.Gameplay.ClassService)
local Constants = require(ReplicatedStorage.Gameplay.Constants)

local NPCSenses = {}

function NPCSenses.GetEnvironmentFeatureVector(npc, abilityDebounces)
	
	local _, distance, direction, _ = NPCSenses.GetEnemyStatus(npc)
	if direction == nil then
		direction = Vector3.new(distance, distance, distance)
	end
	
	local angleToRotate = npc.clientCFrame.LookVector:Angle(direction, Vector3.yAxis)
	
	return {{ 
		1, -- bias
		-- npc angle relative to enemy
		angleToRotate,
		-- distance to enemy
		distance,
		-- abilityDebounces
		abilityDebounces[1] and 1 or 0,
		abilityDebounces[2] and 1 or 0,
		abilityDebounces[3] and 1 or 0,
		abilityDebounces[4] and 1 or 0,
		abilityDebounces[5] and 1 or 0,
	}}
end

function NPCSenses.GetEnemyStatus(npc)
	local closestModel = nil
	local closestDistance = 100000000000000
	local closestDirection = nil
	for _, model in CollectionService:GetTagged(Constants.NPC_TAG) do
		if model == npc:GetModel() or npc.properties.health:Get() <= 0 then
			continue
		end
		local position = ClassService.GetPosition(model)
		local direction = position - ClassService.GetPosition(npc:GetModel())
		local distance = direction.Magnitude
		if distance < closestDistance then
			closestDistance = distance
			closestModel = model
			closestDirection = direction
		end
	end
	
	local health = nil
	if closestModel then
		health = ClassService.GetHealth(closestModel)
	end
	
	return closestModel, closestDistance, closestDirection, health
end

function NPCSenses.GetRewardValue(npc, oldData, debounceTable, damageTable)
	local currentLocation = npc.clientCFrame.Position
	local yRotation = npc.clientCFrame.Rotation.Y
	
	local model, distance, direction, health = NPCSenses.GetEnemyStatus(npc)
	
	local isTooNear = false
	local isInRange = false
	if distance then
		isTooNear = distance <= 2
		isInRange = distance <= 20
	end
	
	local distanceReward = 0
	if isTooNear then
		distanceReward = -1 * 0.1
	elseif not isInRange then
		distanceReward = -1 * 0.1
	elseif isInRange then
		distanceReward = 1 * 0.1
	end
	
	local damageReward = 0
	if oldData.model == model and model ~= nil then
		local healthChanged = health - oldData.health
		if healthChanged < 0 then
			local damage = math.abs(healthChanged)
			damageReward = 10 * damage
		else
			damageReward = 0
		end
	end
	
	local debounceReward = 0
	for i, debounce in debounceTable do
		if damageTable[i] == nil then continue end -- rage no cost
		if debounce == true then
			debounceReward -= 0.1 * damageTable[i].damage / damageTable[i].cd
		end
	end
	
	return distanceReward + damageReward + debounceReward, {
		model = model,
		health = health,
	}
end

return NPCSenses

ThugTraining.lua

local ServerStorage = game:GetService("ServerStorage")
local ReplicatedStorage = game:GetService("ReplicatedStorage")
local DataStoreService = game:GetService("DataStoreService")
local RunService = game:GetService("RunService")

local DataPredict = require(ReplicatedStorage.Modules.DataPredict)
local MatrixL = require(ReplicatedStorage.Modules.MatrixL)
local NPCSenses = require(ServerStorage.DeepLearning.NPCSenses)

local CriticStore = DataStoreService:GetDataStore("Critic")
local ActorStore = DataStoreService:GetDataStore("Actor")

local train = RunService:IsStudio()

local ID = 2

local model = nil

local ThugTraining = {}

function ThugTraining.GetDebounceTable(npc)
	return {
		npc.abilities[1]:GetProperty("AxeKickDebounce"):Get(),
		npc.abilities[2]:GetProperty("PressureStrikeDebounce"):Get(),
		npc.abilities[3]:GetProperty("EarClapDebounce"):Get(),
		npc.abilities[4]:GetProperty("MonsterRageDebounce"):Get(),
		npc.combat:GetProperty("CombatDebounce"):Get(),
	}
end

function ThugTraining.GetDamageTable(npc)
	return {
		{damage = npc.abilities[1].config.damage, cd = npc.abilities[1].config.cd},
		{damage = npc.abilities[2].config.damage, cd = npc.abilities[2].config.cd},
		{damage = npc.abilities[3].config.damage, cd = npc.abilities[3].config.cd},
		nil,
		{damage = npc.combat.config.damage, cd = npc.combat.config.cd},
	}
end

function ThugTraining.GetModel()
	if model ~= nil then
		return model
	end
	
	local ExperienceReplay = DataPredict.ExperienceReplays.UniformExperienceReplay.new(1, 5, 30)
	local MainModel = DataPredict.Models.AdvantageActorCritic.new(60, 0.05, 1)

	MainModel:setExperienceReplay(ExperienceReplay)

	local actor = ThugTraining.BuildActorModel(ID)
	local critic = ThugTraining.BuildCriticModel(ID)
	
	MainModel:setActorModel(actor)
	MainModel:setCriticModel(critic)

	MainModel:setClassesList({"Chase", "UseAbility1", "UseAbility2", "UseAbility3", "UseAbility4", "Combat", "Idle"})
	MainModel:setPrintReinforcementOutput(false)
	
	model = MainModel
		
	return model
end

function ThugTraining.Train(npc)	
	local model = ThugTraining.GetModel()
	
	local oldData = {}

	task.defer(function()
		while npc.properties.health:Get() > 0 do
			local envVec = NPCSenses.GetEnvironmentFeatureVector(npc, ThugTraining.GetDebounceTable(npc))
			local reward, newOldData = NPCSenses.GetRewardValue(npc, oldData, ThugTraining.GetDebounceTable(npc), ThugTraining.GetDamageTable(npc))
			oldData = newOldData
			
			local actionLabel = model:reinforce(envVec, reward)
			if actionLabel == "Chase" then
				npc:Chase()
			elseif actionLabel == "Combat" then
				npc.combat:Start()				
			elseif actionLabel ~= "Idle" then
				npc:UseAbility(tonumber(actionLabel:sub(-1, -1)))
			end
			task.wait(0.1)
		end		
	end)
end

function ThugTraining.SaveModel()
	if model then
		local actorParams = model:getActorModel():getModelParameters()
		local criticParams = model:getCriticModel():getModelParameters()
		local s1 = pcall(function()
			ActorStore:SetAsync(ID, actorParams)
		end)
		local s2 = pcall(function()
			CriticStore:SetAsync(ID, criticParams)
		end)
		print(s1, s2, "saved")
	end
end

function ThugTraining.AutoSaveModel()
	if train then
		task.spawn(function()
			while true do
				task.wait(30)
				ThugTraining.SaveModel()
			end
		end)
	end
end

function ThugTraining.LoadModelParams(ModelDataStore, ID)

	local ModelParameters

	local success = false

	repeat

		success = pcall(function()
			ModelParameters = ModelDataStore:GetAsync(tostring(ID))
		end)

	until success

	--if success then
	--	print("loaded params")
	--end

	return ModelParameters

end

function ThugTraining.BuildActorModel(ID)

	local Model = DataPredict.Models.NeuralNetwork.new(1, 0.001)

	Model:setModelParametersInitializationMode("LeCunUniform")

	Model:addLayer(7, true, 'LeakyReLU')
	Model:addLayer(7, true, 'LeakyReLU')
	Model:addLayer(7, false, 'StableSoftmax')

	Model:setClassesList({"Chase", "UseAbility1", "UseAbility2", "UseAbility3", "UseAbility4", "Combat", "Idle"})


	local ModelParameters = ThugTraining.LoadModelParams(ActorStore, ID)

	Model:setModelParameters(ModelParameters)

	return Model

end

function ThugTraining.BuildCriticModel(ID)
	local Model = DataPredict.Models.NeuralNetwork.new(1, 0.001)

	Model:setModelParametersInitializationMode("LeCunUniform")

	Model:addLayer(7, true, 'LeakyReLU')
	Model:addLayer(4, true, 'LeakyReLU')
	Model:addLayer(1, false, 'Sigmoid')

	Model:setClassesList({1, 2})
	
	local ModelParameters = ThugTraining.LoadModelParams(CriticStore, ID)

	Model:setModelParameters(ModelParameters)

	return Model
end


ThugTraining.AutoSaveModel()
game:BindToClose(function()
	ThugTraining.SaveModel()
end)

return ThugTraining