NPCSenses.lua

local ReplicatedStorage = game:GetService("ReplicatedStorage")
local ServerStorage = game:GetService("ServerStorage")
local CollectionService = game:GetService("CollectionService")

local ClassService = require(ServerStorage.Services.Gameplay.ClassService)
local Constants = require(ReplicatedStorage.Gameplay.Constants)

local NPCSenses = {}

function NPCSenses.GetEnvironmentFeatureVector(npc, abilityDebounces)
	local currentLocation = npc.clientCFrame.Position
	local _, yRotation, _ = npc.clientCFrame:ToEulerAngles()
	
	local _, distance, direction, _ = NPCSenses.GetEnemyStatus(npc)
	if direction == nil then
		direction = Vector3.new(distance, distance, distance)
	end
	
	return {{ 
		1, -- bias
		-- npc pos
		currentLocation.X, 
		currentLocation.Y, 
		currentLocation.Z, 
		-- npc rotation
		yRotation, 
		-- distance to enemy
		distance, 
		-- direction to enemy
		direction.X, 
		direction.Y, 
		direction.Z,
		-- abilityDebounces
		abilityDebounces[1] and 1 or 0,
		abilityDebounces[2] and 1 or 0,
		abilityDebounces[3] and 1 or 0,
		abilityDebounces[4] and 1 or 0,
		abilityDebounces[5] and 1 or 0,
	}}
end

function NPCSenses.GetEnemyStatus(npc)
	local closestModel = nil
	local closestDistance = 100000000000000
	local closestDirection = nil
	for _, model in CollectionService:GetTagged(Constants.NPC_TAG) do
		if model == npc:GetModel() or npc.properties.health:Get() <= 0 then
			continue
		end
		local position = ClassService.GetPosition(model)
		local direction = position - ClassService.GetPosition(npc:GetModel())
		local distance = direction.Magnitude
		if distance < closestDistance then
			closestDistance = distance
			closestModel = model
			closestDirection = direction
		end
	end
	
	local health = nil
	if closestModel then
		health = ClassService.GetHealth(closestModel)
	end
	
	return closestModel, closestDistance, closestDirection, health
end

function NPCSenses.GetRewardValue(npc, oldData, debounceTable, damageTable)
	local currentLocation = npc.clientCFrame.Position
	local yRotation = npc.clientCFrame.Rotation.Y
	
	local model, distance, direction, health = NPCSenses.GetEnemyStatus(npc)
	
	local isTooNear = false
	local isInRange = false
	if distance then
		isTooNear = distance <= 2
		isInRange = distance <= 10
	end
	
	local distanceReward = 0
	if isTooNear then
		distanceReward = -1 * 0.1
	elseif not isInRange then
		distanceReward = -1 * 0.1
	elseif isInRange then
		distanceReward = 0
	end
	
	local damageReward = 0
	if oldData.model == model and model ~= nil then
		local healthChanged = health - oldData.health
		if healthChanged < 0 then
			local damage = math.abs(healthChanged)
			damageReward = 2 * damage
		else
			damageReward = 0
		end
	end
	
	local debounceReward = 0
	for i, debounce in debounceTable do
		if damageTable[i] == nil then continue end -- rage no cost
		if debounce == true then
			debounceReward -= 0.1 * damageTable[i].damage / damageTable[i].cd
		end
	end
	
	return distanceReward + damageReward + debounceReward, {
		model = model,
		health = health,
	}
end

return NPCSenses

ThugTraining.lua

local ServerStorage = game:GetService("ServerStorage")
local ReplicatedStorage = game:GetService("ReplicatedStorage")
local DataStoreService = game:GetService("DataStoreService")
local RunService = game:GetService("RunService")

local DataPredict = require(ReplicatedStorage.Modules.DataPredict)
local MatrixL = require(ReplicatedStorage.Modules.MatrixL)
local NPCSenses = require(ServerStorage.DeepLearning.NPCSenses)

local ParamsStore = DataStoreService:GetDataStore("Params", "v20")
local train = RunService:IsStudio()

local params = nil
local suc = false
while not suc do
	suc, err = pcall(function()
		params = ParamsStore:GetAsync("main")
	end)
	if not suc then
		warn(err)
	end
	print(params)
end
task.spawn(function()
	if train then
		while true do
			local suc, err = pcall(function()
				if params then
					ParamsStore:SetAsync("main", params)
				end
			end)
			if not suc then
				warn(err)
			end
			task.wait(10)
		end
	end
end)

local ThugTraining = {}

function ThugTraining.GetDebounceTable(npc)
	return {
		npc.abilities[1]:GetProperty("AxeKickDebounce"):Get(),
		npc.abilities[2]:GetProperty("PressureStrikeDebounce"):Get(),
		npc.abilities[3]:GetProperty("EarClapDebounce"):Get(),
		npc.abilities[4]:GetProperty("MonsterRageDebounce"):Get(),
		npc.combat:GetProperty("CombatDebounce"):Get(),
	}
end

function ThugTraining.GetDamageTable(npc)
	return {
		{damage = npc.abilities[1].config.damage, cd = npc.abilities[1].config.cd},
		{damage = npc.abilities[2].config.damage, cd = npc.abilities[2].config.cd},
		{damage = npc.abilities[3].config.damage, cd = npc.abilities[3].config.cd},
		nil,
		{damage = npc.combat.config.damage, cd = npc.combat.config.cd},
	}
end

function ThugTraining.Train(npc)
	local DQN = DataPredict.Models.QLearningNeuralNetwork.new(1, 0.01)
	
	DQN:addLayer(13, true, "Sigmoid")
	DQN:addLayer(10, true, "ReLU")
	DQN:addLayer(7, false, "Softmax")

	if params then
		DQN:setModelParameters(params)
	end
	
	DQN:setClassesList({"Chase", "UseAbility1", "UseAbility2", "UseAbility3", "UseAbility4", "Combat", "Idle"})
	DQN:setPrintReinforcementOutput(false)

	local oldData = {}

	task.defer(function()
		while npc.properties.health:Get() > 0 do
			local envVec = NPCSenses.GetEnvironmentFeatureVector(npc, ThugTraining.GetDebounceTable(npc))
			local reward, newOldData = NPCSenses.GetRewardValue(npc, oldData, ThugTraining.GetDebounceTable(npc), ThugTraining.GetDamageTable(npc))
			oldData = newOldData
			
			local actionLabel = DQN:reinforce(envVec, reward)
			
			if actionLabel == "Chase" then
				npc:Chase()
			elseif actionLabel == "Combat" then
				npc.combat:Start()				
			elseif actionLabel ~= "Idle" then
				npc:UseAbility(tonumber(actionLabel:sub(-1, -1)))
			end
			task.wait(0.1)
		end
		
		params = DQN:getModelParameters()
	end)
end

return ThugTraining