Download, unzip and run Elasticsearch, Logstash, Kibana 5.x-6.x on Windows

ElasticStack.ps1

New-Module -Name ElasticStack -Scriptblock {

function WriteLog {
    [CmdletBinding()]
    Param
    (
        [Parameter(Position=0, Mandatory=$true, ValueFromPipelineByPropertyName=$true)]
        [ValidateNotNullOrEmpty()]
        [Alias("M")]
        [string]$Message
    )

    Begin {
    }
    Process {
        $FormattedDate = (Get-Date).ToString("yyyy-MM-dd HH:mm:ssZ")
        $LogMessage = "[$FormattedDate] $Message"
        Write-Host $LogMessage
    }
    End {
    }
}

Set-Alias -Name log -Value WriteLog

function SemanticVersion {
    param(
        [Parameter(Mandatory=$true)]
        $Version
    )

    $Version -match "^(?<Major>\d+)\.(?<Minor>\d+)\.(?<Patch>\d+)(?:\-(?<Prerelease>[\w\-]+))?$" | Out-Null
    $major = [int]$matches['Major']
    $minor = [int]$matches['Minor']
    $patch = [int]$matches['Patch']
    
    if($matches['Prerelease'] -eq $null) {
		$prerelease = @()
	}
    else {
		$prerelease = [string]$matches['Prerelease']
	}

    New-Object PSObject -Property @{ 
        Major = $major
        Minor = $minor
        Patch = $patch
        Prerelease = $prerelease
    }
}

# https://powershellone.wordpress.com/2015/10/27/review-of-methods-to-download-files-using-powershell/
function Download {
    param(
        [Parameter(Mandatory=$true)] 
        $Source, 
        [Parameter(Mandatory=$true)]
        $Destination
    )
    $start = Get-Date
    $job = Start-BitsTransfer -Source $Source -Destination $Destination -DisplayName 'Download' -Asynchronous 
    $destinationPath = Join-Path $Destination ($Source | Split-Path -Leaf)

    while (($job.JobState -eq 'Transferring') -or ($job.JobState -eq 'Connecting')){ 
         filter Get-FileSize {
	        "{0:N2} {1}" -f $(
	        if ($_ -lt 1kb) { $_, 'Bytes' }
	        elseif ($_ -lt 1mb) { ($_/1kb), 'KB' }
	        elseif ($_ -lt 1gb) { ($_/1mb), 'MB' }
	        elseif ($_ -lt 1tb) { ($_/1gb), 'GB' }
	        elseif ($_ -lt 1pb) { ($_/1tb), 'TB' }
	        else { ($_/1pb), 'PB' }
	        )
        }
        $elapsed = ((Get-Date) - $start)
        $averageSpeed = ($job.BytesTransferred * 8 / 1MB) / $elapsed.TotalSeconds
        $elapsed = $elapsed.ToString('hh\:mm\:ss')
        $remainingSeconds = ($job.BytesTotal - $job.BytesTransferred) * 8 / 1MB / $averageSpeed
        $receivedSize = $job.BytesTransferred | Get-FileSize
        $totalSize = $job.BytesTotal | Get-FileSize 
        $progressPercentage = [int]($job.BytesTransferred / $job.BytesTotal * 100)      
        if ($remainingSeconds -as [int]){
            Write-Progress -Activity (" $Source {0:N2} Mbps" -f $averageSpeed) `
                -Status ("{0} of {1} ({2}% in {3})" -f $receivedSize, $totalSize, $progressPercentage, $elapsed) `
                -SecondsRemaining $remainingSeconds `
                -PercentComplete $progressPercentage
        }
    } 
    
    Write-Progress -Activity (" $Source {0:N2} Mbps" -f $averageSpeed) -Status 'Done' -Completed
    Switch($job.JobState){
	    'Transferred' {
            Complete-BitsTransfer -BitsJob $job
            Get-Item $destinationPath | Unblock-File
        }
	    'Error' {
            $job | Format-List
            Write-Error "Download of $Source failed" 
        } 
    }
}

# http://sharepoint.smayes.com/2012/07/extracting-zip-files-using-powershell/
function UnZip {
    param (
        [Parameter(Mandatory=$true)]
        [string]$Zip,
        [Parameter(Mandatory=$true)]
        [string]$Destination
    )
     
    if (!(Test-Path $Destination)) {
        New-Item $Destination -Type Directory
    }

    try {
        $shell = New-Object -ComObject Shell.Application
        $zipFile = $shell.NameSpace($Zip)
        $destinationFolder = $shell.NameSpace($Destination)
        $handle = $destinationFolder.CopyHere($zipFile.Items(), 0x10) #overwrite if exists
    } finally {
        if ($zipFile -ne $null) {
            [void][System.Runtime.InteropServices.Marshal]::ReleaseComObject($zipFile)
        }
        if ($destinationFolder -ne $null) {
            [void][System.Runtime.InteropServices.Marshal]::ReleaseComObject($destinationFolder)
        }
        if ($shell -ne $null) {                
            [void][System.Runtime.InteropServices.Marshal]::ReleaseComObject($shell)
        }
    }
}

function PromptPassword {
    param($prompt)
    while($true) {
        $password = Read-Host -Prompt $prompt -AsSecureString
        if ($password.Length -gt 0) {
            return $password
        }
    }
}

function PlainText {
	param(
		[System.Security.SecureString]
        [Parameter(ValueFromPipeline = $true)]
        $secureString
	)

	$bstr = [Runtime.InteropServices.Marshal]::SecureStringToBSTR($secureString);
	try {
		return [Runtime.InteropServices.Marshal]::PtrToStringBSTR($bstr);
	}
	finally {
		[Runtime.InteropServices.Marshal]::FreeBSTR($bstr);
	}
}

function SetUpUsers {
    $contentType = "application/json"
    $password = $bootstrapPassword
    $localhost = "http://localhost:9200"

    # wait for the node to become available
    while($true) {
        try {
            irm $localhost -ContentType $contentType
        }
        catch {
            if ($_.Exception.InnerException -ne $null -and $_.Exception.InnerException -is [System.Net.Sockets.SocketException]) {
                sleep -Milliseconds 500
            } else {
                try {
                    $response = $_ | ConvertFrom-Json
                    if ($response -and $response.status -and ($response.status -eq 401)) {
                        break
                    }  
                } catch {} 
            }
        }
    }

    $script:elasticUser = PromptPassword "Enter 'elastic' user password"
    $script:kibanaUser = PromptPassword "Enter 'kibana' user password"
    
    $users = [ordered]@{
        elastic = $elasticUser
        kibana = $kibanaUser
    }

    # logstash_system user exists only in 5.2.0+
    if ($versionEqualOrGreaterThan520) {
        $script:logstashUser = PromptPassword "Enter 'logstash_system' user password"
        $users["logstash_system"] = $logstashUser
    }

    $users.GetEnumerator() | %{
        log "updating password for built-in user $($_.Key)"
        $credential = New-Object System.Management.Automation.PSCredential "elastic", $password
        irm "$localhost/_xpack/security/user/$($_.Key)/_password" -Method Put -ContentType $contentType `
            -Body "{`"password`":`"$($_.Value | PlainText)`"}" -Credential $credential
        if ($_.Key -eq "elastic") {
            $password = $_.Value
        }
        log "updated password for built-in user $($_.Key)"
    }
}

function SetupXPack {
    param(
        $BinDir,
        $Product
    )

    switch($Product) {
        'elasticsearch' {       
            $keystoreBat = [IO.Path]::Combine($BinDir, 'elasticsearch-keystore.bat')
            # keystore only exists for later Elasticsearch 5.x versions
            if (Test-Path $keystoreBat) {
                
                $keystore = [IO.Path]::Combine($BinDir, '..', 'config', 'elasticsearch.keystore')
                $bootstrapKey = 'bootstrap.password'
                $set = $false
                if (Test-Path $keystore) {
                    log "keystore exists. Checking for $bootstrapKey"
                    $keys = & $keystoreBat | Out-Null
                    if ($keys -contains $bootstrapKey) {
                        $set = $true
                        log "$bootstrapKey exists in keystore"
                    } else {
                        log "$bootstrapKey does not exist in keystore"
                    }
                }

                # setup bootstrap password
                $prompt = "Enter bootstrap password"
                if ($set) {
                    $prompt = "Enter existing bootstrap password"
                }
                $script:bootstrapPassword = PromptPassword $prompt

                if (-not $set) {
                    log "adding $bootstrapKey to keystore"
                    $info = New-Object System.Diagnostics.ProcessStartInfo
                    $info.FileName = $keystoreBat
                    $info.Arguments = "add $bootstrapKey -xf"
                    $info.UseShellExecute = $false
                    $info.RedirectStandardInput = $true
                    $info.CreateNoWindow = $true
                    $process = [System.Diagnostics.Process]::Start($info)
                    $p = $bootstrapPassword | PlainText
                    $process.StandardInput.WriteLine($p)
                    $process.StandardInput.Close()
                    $process.WaitForExit()
                    if ($process.ExitCode -ne 0) {
                        throw "Process to set $bootstrapKey exited with $($process.ExitCode)"
                    }
                    $process.Close()
                    log "added $bootstrapKey to keystore"
                }
            }
        }
        'kibana' {
            # setup kibana user in config
            $configFile = [IO.Path]::Combine($BinDir, '..', 'config', 'kibana.yml')
            log "adding kibana username and password to kibana.yml file at $configFile"
            $content = [IO.File]::ReadAllLines($configFile, [Text.Encoding]::UTF8)
            for($x =0; $x -lt $content.Length; $x++) {
                $line = $content[$x]
                if ($line -contains '#elasticsearch.username: "user"') {
                    $content[$x] = 'elasticsearch.username: "kibana"'
                } elseif($line -contains '#elasticsearch.password: "pass"') {
                    $content[$x] = "elasticsearch.password: `"$($kibanaUser | PlainText)`""
                }
            }         
            [IO.File]::WriteAllLines($configFile, $content, [Text.Encoding]::UTF8)
            log "added kibana username and password to kibana.yml file at $configFile"
        }
        'logstash' {
            if ($versionEqualOrGreaterThan520) {
                # setup logstash_system user in config
                $configFile = [IO.Path]::Combine($BinDir, '..', 'config', 'logstash.yml')
                if (!(Select-String -Path $configFile -Pattern "xpack.monitoring.elasticsearch.username")) {
                    log "adding logstash_system username and password to logstash.yml file at $configFile"
                    $monitoringConfig = @(
                        "xpack.monitoring.elasticsearch.username: `"logstash_system`"", 
                        "xpack.monitoring.elasticsearch.password: `"$($logstashUser | PlainText)`"")
                    [IO.File]::AppendAllLines([string]$configFile, [string[]]$monitoringConfig, [Text.Encoding]::UTF8)
                    log "added logstash_system username and password to logstash.yml file at $configFile"
                }
            }
        }
    }
}

function DownloadInstallPlugins {
    param(
        $BinDir,
        $Product,
        $Plugin,
        $PluginArguments
    )

    $pluginBatFile = [IO.Path]::Combine($BinDir, "$Product-plugin.bat")
    $pluginsDir = [IO.Path]::Combine($BinDir, "..", "plugins")     
    $installedPlugins = @()
    # plugins dir needs to exist for earlier Elasticsearch 5.x
    if (Test-Path $pluginsDir) {
        log "checking currently installed plugins for $product"
        # handle kibana plugin list, which uses <plugin>@<version> format  
        $installedPlugins = & $pluginBatFile 'list' | %{ $_.Split('@')[0].Trim() } | ?{ $_ -ne "" }
        if ($installedPlugins) {
            log "installed plugins for $product - $([String]::Join(',', $installedPlugins))"
        } else {
            log "no installed plugins for $product"
        }
    }

    $pluginsToInstall = $Plugin | ?{ $installedPlugins -notcontains $_ }  
    foreach($p in $pluginsToInstall) {
        $pluginCommand = @("install", $p)
        if ($PluginArguments) {
            $pluginCommand = $pluginCommand + $PluginArguments
        }
        log "installing $p for $product"
        Start-Process $pluginBatFile -ArgumentList $pluginCommand -Wait 
        log "installed $p for $product"
    }

    if ($pluginsToInstall -contains "x-pack") {
        log "setting up X-Pack for $product"
        SetupXPack -BinDir $BinDir -Product $Product
        log "set up X-Pack for $product"
    }
    $pluginsToInstall
}

function DownloadUnzipAndStart {
    param(
        $Source, 
        $Destination, 
        $Port, 
        $StartArguments, 
        $Plugin, 
        $PluginArguments
    )

    $Source = "https://artifacts.elastic.co/downloads/$Source"
    $zipName = [IO.Path]::GetFileName($Source)
    $zipFile = Join-Path $Destination $zipName
    $product = $zipName.Split('-')[0]
    $unzippedDir = Join-Path $Destination $([IO.Path]::GetFileNameWithoutExtension($zipName))

    if (!(Test-Path $zipFile)) {
        log "downloading $product from $Source"
        Download -Source $Source -Destination $Destination
        log "downloaded $product to $Destination"
    } else {
        log "using existing zip archive for $product at $zipFile"
    }

    if (!(Test-Path $unzippedDir)) {
        log "unzipping $product to $Destination"
        UnZip -Zip $zipFile -Destination $Destination
        log "unzipped $product to $Destination"
    } else {
        log "zip archive for $product already unzipped at $unzippedDir"
    }

    $binDir = [IO.Path]::Combine($Destination, $([IO.Path]::GetFileNameWithoutExtension($zipName)), "bin")
    
    if ($Plugin) {
        $Plugin = $Plugin | sort -uniq | %{ $_.ToLowerInvariant() }
        $installedPlugins = DownloadInstallPlugins -BinDir $binDir -Product $product -Plugin $Plugin -PluginArguments $PluginArguments
    }

    $batFile = [IO.Path]::Combine($binDir, "$product.bat")
    if (Test-Path $batFile) {
        log "starting $product process"
        if ($StartArguments) {
            Start-Process $batFile -ArgumentList $StartArguments
        } else {
            Start-Process $batFile
        }
        log "started $product process"

        if ($installedPlugins -and $installedPlugins -contains "x-pack" -and $product -eq "elasticsearch") {
            SetupUsers
        }

        $uri = "http://localhost:$Port"
        log "opening browser window to $uri for $product"
        Start-Process $uri
        log "opened browser window to $uri for $product"
    }
}

function Install-ElasticStack {
    <#
    .Synopsis
        Get up and running with Elasticsearch, Kibana and Logstash on Windows
    .Description
        Downloads the zip archives of Elasticsearch, Kibana and Logstash, unzips and starts them. Launches
        a browser window to view the running process
    .Example
        & .\ElasticStack.ps1
    .Example
        & .\ElasticStack.ps1 -Destination "C:\temp" -Version "6.1.2"
    .Example
        & .\ElasticStack.ps1 -Product Elasticsearch,Kibana
    .Parameter Destination
        destination directory for Elasticsearch, Logstash and Kibana. Defaults to $env:USERPROFILE\Downloads
    .Parameter Product
        the Elastic stack products to download. Defaults to Elasticsearch,Kibana,Logstash
    .Parameter Version
        the Elastic stack version. Defaults to 6.2.1
    .Parameter ElasticsearchPlugin
        the Elasticsearch plugins to install. Defaults to x-pack
    .Parameter KibanaPlugin
        the Kibana plugins to install. Defaults to x-pack
    .Parameter LogstashPlugin
        the Logstash plugins to install. Defaults to x-pack
    .Parameter LogstashConfig
        the path to the logstash configuration file to start Logstash with. If none supplied, uses an inline configuration to read from standard input and write to standard output
    #>
    [CmdletBinding()]
    Param(
        [Parameter(Mandatory=$false)]
        [ValidateScript({if ($_){ Test-Path $_ -PathType Container}})]
        [string] $Destination = "$env:USERPROFILE\Downloads",

        [Parameter(Mandatory=$false)]
        [ValidateSet('Elasticsearch','Kibana','Logstash',IgnoreCase = $false)]
        [string[]] $Product = @('Elasticsearch','Kibana','Logstash'),

        [Parameter(Mandatory=$false)]
        [ValidatePattern("^([5-9]|1\d+)\.\d+\.\d+(?:\-[\w\-]+)?$")]
        [string] $Version = "6.2.1",

        [Parameter(Mandatory=$false)]
        [string[]] $ElasticsearchPlugin = @("x-pack"),

        [Parameter(Mandatory=$false)]
        [string[]] $KibanaPlugin = @("x-pack"),

        [Parameter(Mandatory=$false)]
        [string[]] $LogstashPlugin = @("x-pack"),

        [Parameter(Mandatory=$false)]
        [ValidateScript({if ($_){ Test-Path $_ -PathType Leaf}})]
        [string] $LogstashConfig
    )

    $ErrorActionPreference = "Stop"
    $bootstrapPassword = "changeme" | ConvertTo-SecureString -AsPlainText -Force
    $elasticUser = "elastic" | ConvertTo-SecureString -AsPlainText -Force
    $kibanaUser = "kibana" | ConvertTo-SecureString -AsPlainText -Force
    $logstashUser = "logstash" | ConvertTo-SecureString -AsPlainText -Force
    $semanticVersion = SemanticVersion $Version
    $versionEqualOrGreaterThan520 = $semanticVersion.Major -gt 5 -or $semanticVersion.Minor -ge 2

    $Product = $Product | sort -uniq

    # kibana download url
    if ([int]::Parse($Version[0]) -gt 5) {
        $64bitSuffix = "_64"
    }

    # logstash configuration
    if (!($LogstashConfig)) {
        $logstashStartArguments = "-e 'input { stdin { } } output { stdout {} }'"
    } else {
        $logstashStartArguments = "-f '$LogstashConfig'"
    }

    # if any product is installing x-pack, install it for all
    $xpack = "x-pack"
    if ($ElasticsearchPlugin -contains $xpack -or $KibanaPlugin -contains $xpack -or $LogstashPlugin -contains $xpack) {
        $ElasticsearchPlugin = $ElasticsearchPlugin + $xpack
        $KibanaPlugin = $KibanaPlugin + $xpack
        $LogstashPlugin = $LogstashPlugin + $xpack
    }

    $products = [ordered]@{
        "Elasticsearch" = @{
            Destination = $Destination
            Source = "elasticsearch/elasticsearch-$Version.zip"
            Port = 9200
            Plugin = $ElasticsearchPlugin
            PluginArguments = "--batch"
        }
        "Kibana" = @{
            Source = "kibana/kibana-$Version-windows-x86$64bitSuffix.zip"
            Destination = $Destination
            Port = 5601      
            Plugin = $KibanaPlugin
        }
        "Logstash" = @{
            Source = "logstash/logstash-$Version.zip"
            Destination = $Destination
            Port = 9600
            StartArguments = $logstashStartArguments
            Plugin = $LogstashPlugin
        }
    }

    foreach($p in $products.GetEnumerator()) {
        if ($Product.Contains($p.Key)) {
            $a = $p.Value
            DownloadUnzipAndStart @a
        }
    }

}

Set-Alias install -value Install-ElasticStack
Export-ModuleMember -Function 'Install-ElasticStack' -Alias 'install'

}

To run

. { iwr -useb https://gist.githubusercontent.com/russcam/ee5293c7ae5bf1fa22de13cbbd2d28c4/raw/90d8f4f78ae4be1b1fab8796bcb782607d8f60d0/ElasticStack.ps1 } | iex; Install-ElasticStack -Product Elasticsearch,Kibana -Version 6.2.1

Modify the Install-ElasticStack parameters to your needs.