markoutso · January 18, 2021 23:07
diff --git a/Readme.md b/Readme.md
diff --git a/functions.ps1 b/functions.ps1
 function Get-GitlabIssues {
 	[CmdletBinding()]
 	param (
 		[parameter(Mandatory=$true)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$AccessToken,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$ContainerPath)
 	Begin {
 		$issues = $null;
 		$result = $null;
 		$page = 1;
 		Write-Progress -Activity "Get-GitlabIssues" -Status "0 Complete:" -PercentComplete 0;
 	}
 	Process {
 		$total = (Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues_statistics" -Headers @{ 'Authorization' = "Bearer $AccessToken" }).statistics.counts.all;
 		while($result -ne 0) {
 			$result = Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" }
 			$issues += $result;
 			if($issues.Count  -eq 0) {
 				$count = 1;
 			} else {
 				$count = $issues.Count;
 			}
 			$percent = $count/$total*100;
 			$page ++;
 			Write-Progress -Activity "Get-GitlabIssues" -Status "$percent% Complete:" -PercentComplete $percent;
 		}
 	}
 	End {
 		return $issues;
 	}
 }


 function Get-GitlabComments {
 	[CmdletBinding()]
 	param (
 		[parameter(Mandatory=$true)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$AccessToken,
 		[parameter(Mandatory=$true, ValueFromPipeline)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$NotesLink)
 	Begin {
 	}
 	Process {
 		try{
 			Invoke-RestMethod -Method Get -Uri "$NotesLink`?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" };
 		} catch {
 			$issueIid = $NotesLink.Split("/") | Select-Object -Last 2 | Select-Object -First 1
 			Write-Error -Message "Failed to get comments for issue: $issueIid" -ErrorId 500 -TargetObject $issueIid -Category InvalidResult;
 		}
 	}
 	End {
 	}
 }


 function Get-GitLabAttachment {
 	[CmdletBinding(
 		SupportsShouldProcess,
 		ConfirmImpact="Medium"
 	)]
 	param (
 		[parameter(Mandatory)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$AccessToken,
 		[parameter(Mandatory)]
 		[System.String]
 		$Root,
 		[parameter(Mandatory)]
 		[System.String]
 		$Issue,
 		[parameter(Mandatory)]
 		[System.String]
 		$File)
 	Begin {
 	}
 	Process {
 		$path = Join-Path -Path $Root -ChildPath $Issue | Join-Path -ChildPath "Attachments";
 		New-Item -Path $path -Type Directory -Force | Out-Null;
 		$path = $path | Join-Path -ChildPath ("~"+([string]::Join("~", ($File.Split("/") | Select-Object -Last 3))));
 		if(-Not (Test-Path -Path $path)) {
 			Try {
 				Invoke-WebRequest -OutFile $path -Uri $File -Headers @{ 'Authorization' = "Bearer $AccessToken" };
 			} Catch {
 				Write-Error -Message "Failed to get attachment: $File for issue: $Issue" -ErrorId 500 -TargetObject $Issue -Category InvalidResult;
 			}
 		}
 	}
 	End {
 	}
 }


 function Add-DevopsWorkItem {
 	[CmdletBinding(
 		SupportsShouldProcess=$true,
 		ConfirmImpact="Medium"
 	)]
 	param (
 		[parameter(Mandatory=$true)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$AccessToken,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Organization,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Project,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		[ValidateSet("Epic", "Feature", "User Story", "Bug", "Issue", "Task")]
 		$Type,
 		[System.Int32]
 		$Id,
 		[parameter(Mandatory=$true,ValueFromPipeline)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Data,
 		[Switch]
 		$Test)
 	Begin {
 		$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
 		$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workitems/`$$($Type)?api-version=5.1&bypassRules=true&suppressNotifications=true";
 	}
 	Process {
 		if ($PSCmdlet.ShouldProcess($Data)) {
 			Try {
 				if($Test) {
 					Invoke-RestMethod -Method Patch -Uri $uri+"&validateOnly=true" -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
 				} else {
 						Invoke-RestMethod -Method Patch -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
 				}
 			} Catch {
 				Write-Error -Message "Failed to add issue for issue: $($Id)" -ErrorId 500 -TargetObject $Data -Category InvalidResult;
 			}
 		}
 	}
 	End {
 	}
 }


 function Add-DevopsComment {
 	[CmdletBinding(
 		SupportsShouldProcess=$true,
 		ConfirmImpact="Medium"
 	)]
 	param (
 		[parameter(Mandatory=$true)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$AccessToken,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Organization,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Project,
 		[ValidateNotNullOrEmpty()]
 		[System.Int32]
 		$Id,
 		[parameter(Mandatory=$true,ValueFromPipeline)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Data)
 	Begin {
 		$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
 		$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workItems/$Id/comments?api-version=5.1-preview.3";
 	}
 	Process {
 		if ($PSCmdlet.ShouldProcess($Data)) {
 			Try{
 				Invoke-RestMethod -Method Post -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json' -Body $Data
 			} Catch {
 				Write-Error -Message "Failed to add comments for issue: $($Id)" -ErrorId 500 -TargetObject $Id -Category InvalidResult;
 			}
 		}
 	}
 	End {
 	}
 }


 function Add-DevopsAttachment {
 	[CmdletBinding(
 		SupportsShouldProcess=$true,
 		ConfirmImpact="Medium"
 	)]
 	param (
 		[parameter(Mandatory=$true)]
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$AccessToken,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Organization,
 		[ValidateNotNullOrEmpty()]
 		[System.String]
 		$Project,
 		[parameter(Mandatory=$true,ValueFromPipeline)]
 		[System.IO.FileInfo]
 		$File)
 	Begin {
 		$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)));
 		$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/attachments?api-version=5.0";
 	}
 	Process {
 		if ($PSCmdlet.ShouldProcess($FileName)) {
 			Try{
 				$image = [System.IO.File]::ReadAllBytes($File.FullName);
 				Invoke-RestMethod -Uri $uri+"&fileName=$([System.Web.HttpUtility]::UrlEncode($File.Name))" -Method Post -Headers @{ Authorization=("Basic {0}" -f $basicAuth) } -ContentType application/json -Body $image;
 			} Catch {
 				Write-Error -Message "Failed to add attachments for issue: $($File.FullName)" -ErrorId 500 -TargetObject $File -Category InvalidResult;
 			}
 		}
 	}
 	End {
 	}
 }


 function Convert-IssueToWorkItem {
 	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
 	[CmdletBinding()]
 	[OutputType([string])]
 	param (
 		[parameter(Mandatory=$true)]
 		$RootIterationPath,
 		[parameter(Mandatory=$true,ValueFromPipeline)]
 		$Issue
 	)
 	Begin {
 		Import-Module MarkdownToHtml -Force;
 		$userMap = @{
 			#https://gitlab.com/api/v4/groups/mygroupid/members GitLabUserUserId="AzureDevOpsDisplayName",
 			123 ="John Doe";
 		};
 	}
 	Process {
 		if([String]::IsNullOrWhiteSpace($Issue.description)) {
 			$content = "`" `"";
 		} else {
 			$content = ((Convert-MarkdownToHTMLFragment -Markdown ($Issue.description | Format-BadCharacters)).HtmlFragment).Replace("₹", "&#x20b9;").Replace("'", "&#39;") | ConvertTo-Json;
 		}
 		$title = [System.Web.HttpUtility]::HtmlEncode(($Issue.title | Format-BadCharacters));
 		if($title.Length -gt 255){$title = $title.Substring(0,247);}
 		$author = $($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.author.id}).Value;
 		$assignee = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
 		$closedby = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.closed_by.id}).Value;
 		$tags = [String]::Join(";", $Issue.labels+$Issue.state);
 		if($Issue.milestone) {
 			$iteration = $RootIterationPath+"\\"+$Issue.milestone.title.Replace("?", "").Replace("&", "and");
 		} else {
 			$iteration = $RootIterationPath;
 		}
 		if($null -ne $Issue.assignee){
 			$user = ($userMap.GetEnumerator() | Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
 			$assignee = ",
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.AssignedTo`",
 		`"from`": null,
 		`"value`": `"$($user)`"
 	}"
 		} else {
 			$assignee = "";
 		}
 		$type = @("User Story", "Bug")[$Issue.labels.Contains("Bug")];
 		if($type -eq "Bug") {
 			$body = ",
 			{
 				`"op`": `"add`",
 				`"path`": `"/fields/Microsoft.VSTS.TCM.ReproSteps`",
 				`"from`": null,
 				`"value`": $content
 			}"
 		} else {
 			$body = ",
 			{
 				`"op`": `"add`",
 				`"path`": `"/fields/System.Description`",
 				`"from`": null,
 				`"value`": $content
 			}"
 		}
 		if($Issue.state -eq "closed"){
 			$closed = ",
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/Microsoft.VSTS.Common.ClosedBy`",
 		`"from`": null,
 		`"value`": `"$($closedby)`"
 	},
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/Microsoft.VSTS.Common.ClosedDate`",
 		`"from`": null,
 		`"value`": `"$($Issue.closed_at)`"
 	}";
 		} else {
 			$closed = "";
 		}
 return "[
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.IterationPath`",
 		`"from`": null,
 		`"value`": `"$($iteration)`"
 	},
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.Title`",
 		`"from`": null,
 		`"value`": `"$($Issue.iid): $($title)`"
 	}$($body),
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.State`",
 		`"from`": null,
 		`"value`": `"$(@{"opened"="New"; "closed" = "New"}[$Issue.state])`"
 	},
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.Tags`",
 		`"from`": null,
 		`"value`": `"$($tags)`"
 	},
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.CreatedBy`",
 		`"from`": null,
 		`"value`": `"$($author)`"
 	},
 	{
 		`"op`": `"add`",
 		`"path`": `"/fields/System.CreatedDate`",
 		`"from`": null,
 		`"value`": `"$($Issue.created_at)`"
 	}$($assignee)$($closed)
 ]";
 	}
 	End {
 	}
 }

 function Convert-NoteToComment {
 	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
 	[CmdletBinding()]
 	[OutputType([string])]
 	param(
 		[parameter(Mandatory,ValueFromPipeline)]
 		$Note
 	)
 	Begin {
 		Import-Module MarkdownToHtml -Force;
 	}
 	Process {
 		$header = "<p><b>$($Note.created_at.ToLocalTime().ToString('yyyy-MM-dd HH:mm:ss')) by $($Note.author.name)</b></p>";
 		if([String]::IsNullOrWhiteSpace($Note.body)) {
 			$content = "`" `"";
 		} else {
 			$content = ($header + (Convert-MarkdownToHTMLFragment -Markdown ($Note.body | Format-BadCharacters)).HtmlFragment).Replace("₹", "&#x20b9;") | ConvertTo-Json;
 		}
 		return "{ `"text`": $content }";
 	}
 	End {
 	}
 }

 function Format-BadCharacters {
 	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
 	[CmdletBinding()]
 	[OutputType([string])]
 	param (
 		[parameter(
 			Mandatory,
 			ValueFromPipeline,
 			ValueFromPipelineByPropertyName,
 			Position = 0)]
 		[string] $Input
 	)
 	Process {
 		return $Input.Replace("`“", "`"").Replace("`”", "`"").Replace(" ", " ").Replace("ä", "a").Replace("ë", "e");
 	}
 }
diff --git a/process.ps1 b/process.ps1
 #Requires -PSEdition Core
 #Requires -Modules @{ ModuleName="GitLab"; ModuleVersion="0.0"; }, @{ ModuleName="AzureDevops"; ModuleVersion="0.0"; }, @{ ModuleName="GitLabDevOpsMigration"; ModuleVersion="0.0"; }, @{ ModuleName="MarkdownToHtml"; ModuleVersion="2.0.0"}

 throw "This is purely here so you dont accidentaly trigger this script, it is not designed to run end to end in one step, Please read the instructions"

 ### Setup working directory
 $root = Join-Path -Path $(Get-Location) -ChildPath "MigrationWorkSpace"
 New-Item -Path $root -Type Directory -Force;

 ### Step 1 - Download all GitLab Issues
 $issues = Get-GitlabIssues -AccessToken $GitLabAccessToken -ContainerPath $GitlabContainerPath;

 ### Step 2 - Had to run this a few times so I always kept track of migrated ids in this file
 $MigratedIssueSucessPath = $root | Join-Path -ChildPath "GtDIssueSucess.txt";
 if(Test-Path -Path $MigratedIssueSucessPath) {
 	$migratedIssueIds = Get-Content $MigratedIssueSucessPath | ForEach-Object{[int64]::Parse($_)}
 } else { $migratedIssueIds = @()}
 $issuesToMigrate = $issues | Where-Object{ -not $migratedIssueIds.Contains($_.id)}


 ### Step 3 - Save all issues into a folder
 $issuesToMigrate | ForEach-Object {
 	$folder = Join-Path -Path $root -ChildPath $_.id;
 	New-Item -Path $folder -Type Directory -Force | Out-Null;
 	$_ | ConvertTo-Json | Out-File -FilePath (Join-Path -Path $folder -ChildPath issue.json) -Encoding utf8 -Force};

 ### Step 4 - Get Unique Milestones and their status
 ### These need to be created as sprints in devops manualy, I only had to create about a dozen so was easy enough
 $issuesToMigrate | Select-Object -ExpandProperty milestone | Select-Object title, state -Unique | Sort-Object state, name

 ### Step 5 - Download Comments
 $total = $issuesToMigrate.Count;
 $counter = 1;
 $issuesToMigrate | Sort-Object id | ForEach-Object -Begin {
 		Write-Progress -Activity "Get-GitlabComments" -Status "0% Complete:" -PercentComplete 0;
 	} -Process {
 		$percent = $counter/$total*100;
 		$counter++;
 		Write-Progress -Activity "Get-GitlabComments" -Status "$percent% Complete: $($_.id)" -PercentComplete $percent;
 			$folder = Join-Path -Path $root -ChildPath $_.id;
 			Get-GitlabComments -AccessToken $GitLabAccessToken -NotesLink $_._links.notes |
 			ConvertTo-Json |
 			Out-File -FilePath (Join-Path -Path $folder -ChildPath notes.json) -Encoding utf8 -Force;
 	} -End {
 		Write-Progress -Activity "Get-GitlabComments" -Status "100% Complete:" -PercentComplete 100 -Completed
 	}

 ### Step 6 - Get all attachment links
 $files = @();
 #### Get attachment links from issues
 Get-ChildItem -Path $root |
 	ForEach-Object { $id = $_.Name;
 		Get-ChildItem -Path $_.FullName -Filter issue.json |
 		Get-Content -Raw |
 		ConvertFrom-Json |
 		Where-Object   { -not [String]::IsNullOrWhiteSpace($_.description) } |
 		ForEach-Object {
 			$urlParts = $_.web_url.Split("/");
 			$rootPath = [string]::Join("/", ($urlParts | Select-Object -First ($urlParts.Count-2)));
 			$_.description.Split("`n")  |
 			ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} |
 			Where-Object   { $_.Success } |
 			Select-Object -ExpandProperty Groups |
 			Where-Object   { -Not $_.Value.StartsWith("]")} |
 			Select-Object -ExpandProperty Value |
 			ForEach-Object {
 				$files+=New-Object PSObject -Property @{
 					issue=$id; 
 					file=$rootPath+$_;} 
 			}
 		}
 	}

 #### Get attachment links from comments
 Get-ChildItem -Path $root |
 	ForEach-Object { 
 		$id = $_.Name;
 		$urlParts = (Get-ChildItem -Path $_.FullName -Filter issue.json |
 		Get-Content -Raw |
 		ConvertFrom-Json).web_url.Split("/");
 		$rootPath = [string]::Join("/", ($urlParts | Select-Object -First ($urlParts.Count-2)));
 		Get-ChildItem -Path $_.FullName -Filter notes.json |
 		Get-Content -Raw |
 		ConvertFrom-Json |
 		Where-Object   { -not [String]::IsNullOrWhiteSpace($_.body) } |
 		ForEach-Object { $_.body.Split("`n") } |
 		ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} |
 		Where-Object   { $_.Success } |
 		Select-Object -ExpandProperty Groups |
 		Where-Object   { -Not $_.Value.StartsWith("]")} |
 		Select-Object -ExpandProperty Value |
 		ForEach-Object {
 			$files+=New-Object PSObject -Property @{
 				issue=$id; 
 				file=$rootPath+$_;} }
 	}



 ### Step 7 - Downlaod the attachments from gitlab NOTE: This can take while
 $total = $files.Count;
 $counter = 1;
 $files | ForEach-Object -Begin {
 		Write-Progress -Activity "Get-GitlabAttachments" -Status "0% Complete:" -PercentComplete 0;
 	} -Process {
 		$percent = $counter/$total*100;
 		$counter++;
 		Write-Progress -Activity "Get-GitlabAttachments" -Status "$percent% Complete: $($_.issue)" -PercentComplete $percent;
 		Get-GitLabAttachment -AccessToken $GitLabAccessToken -Root $root -Issue $_.Issue -File $_.File;
 	} -End {
 		Write-Progress -Activity "Get-GitlabAttachments" -Status "100% Complete:" -PercentComplete 100 -Completed
 	}

 ### Step 8 - Upload and map attachments to offlined notes and comments
 $total = (Get-ChildItem -Path $root).Count;
 $counter = 1;
 Get-ChildItem -Path $root |
 	ForEach-Object -Begin {
 		Write-Progress -Activity "Map-Attachments" -Status "0% Complete:" -PercentComplete 0;
 	} -Process {
 		$percent = $counter/$total*100;
 		$counter++;
 		Write-Progress -Activity "Map-Attachments" -Status "$percent% Complete: $($_.Name)" -PercentComplete $percent;
 			Push-Location -Path $_.FullName;
 				if(Test-Path -Path Attachments) {
 					$replace = Get-ChildItem -Path Attachments | ForEach-Object {
 						$name = $_ | Select-Object -Property @{Name="Name";Expression={$_.Name.Replace("~","/")}} | Select-Object -ExpandProperty Name;
 						$upld = Add-DevopsAttachment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -File $_;
 						Write-Verbose "$($name) $($upld.url)" -Verbose;
 						@{Old=$name; New=$upld.url};
 					}

 					$replace | ForEach-Object {
 						$old = $_.Old;
 						$new = $_.New;
 						Get-ChildItem -File | ForEach-Object {
 							$fileName = $_.FullName;
 							$content = Get-Content $_ | ForEach-Object {
 								$_.Replace($old, $new);
 							};
 							$content | Out-File -FilePath $fileName -Encoding utf8 -Force;
 						}
 					}
 				}
 			Pop-Location;
 	} -End {
 		Write-Progress -Activity "Map-Attachments" -Status "100% Complete:" -PercentComplete 100 -Completed
 	}
 ###  
 # At this stage I had to do some manual fixes sometimes, important at this stage to re-load all the issues and commments from the filesystem as they have the updated data after uploading all the attachments
 ###
 $issuesToMigrate = Get-ChildItem -Path $root -Filter issue.json -Recurse | Get-Content -Raw | ConvertFrom-Json

 ### Step 9 - Issue + Comment Upload
 $failed = @();
 $total = $issuesToMigrate.Count;
 $test=$true; # This will allow you to validate before import, set to false for actual import
 $counter = 1;
 #### The Sort helped me judge how far into the process I was
 $issuesToMigrate | Sort-Object id |
 	ForEach-Object -Begin {
        Write-Progress -Activity "Create Work Items and comments" -Status "0% Complete:" -PercentComplete 0;
    } -Process {
        $percent = $counter/$total*100;
        $counter++;
 		$id = $_.id;
 		### Note the new bug template needs to map the content to another field other than description, think it was called reposteps or similar
 		$type = @("User Story", "Bug")[$_.labels.Contains("Bug")];
        Write-Progress -Activity "Create Work Items and comments" -Status "$percent% Complete: $($id)" -PercentComplete $percent;
 		$_ | Convert-IssueToWorkItem -RootIterationPath "Squad.Umenit" | ForEach-Object{
 			$workItem = $null;
 			$workItem = Add-DevopsWorkItem -Test:$test -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $id -Type $type -Data $_;
 			if($null -eq $workItem){
 				$failed += @{$id=$_}
 			} else {
 				if(-not $test) {
 					$comments = Get-Content -Path ($root | Join-Path -ChildPath $id | Join-Path -ChildPath "notes.json") -Raw | ConvertFrom-Json | Sort-Object -Property created_at;
 					$comments.GetEnumerator() | Convert-NoteToComment | Add-DevopsComment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $workItem.id | Out-Null;
 				}
 			}
 		}
    } -End {
        Write-Progress -Activity "Create Work Items and comments" -Status "100% Complete:" -PercentComplete 100 -Completed
    }
	function Get-GitlabIssues {
	[CmdletBinding()]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$ContainerPath)
	Begin {
	$issues = $null;
	$result = $null;
	$page = 1;
	Write-Progress -Activity "Get-GitlabIssues" -Status "0 Complete:" -PercentComplete 0;
	}
	Process {
	$total = (Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues_statistics" -Headers @{ 'Authorization' = "Bearer $AccessToken" }).statistics.counts.all;
	while($result -ne 0) {
	$result = Invoke-RestMethod -Method Get -Uri "https://gitlab.com/api/v4/$ContainerPath/issues?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" }
	$issues += $result;
	if($issues.Count -eq 0) {
	$count = 1;
	} else {
	$count = $issues.Count;
	}
	$percent = $count/$total*100;
	$page ++;
	Write-Progress -Activity "Get-GitlabIssues" -Status "$percent% Complete:" -PercentComplete $percent;
	}
	}
	End {
	return $issues;
	}
	}


	function Get-GitlabComments {
	[CmdletBinding()]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[parameter(Mandatory=$true, ValueFromPipeline)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$NotesLink)
	Begin {
	}
	Process {
	try{
	Invoke-RestMethod -Method Get -Uri "$NotesLink`?page=$page&per_page=100" -Headers @{ 'Authorization' = "Bearer $AccessToken" };
	} catch {
	$issueIid = $NotesLink.Split("/") \| Select-Object -Last 2 \| Select-Object -First 1
	Write-Error -Message "Failed to get comments for issue: $issueIid" -ErrorId 500 -TargetObject $issueIid -Category InvalidResult;
	}
	}
	End {
	}
	}


	function Get-GitLabAttachment {
	[CmdletBinding(
	SupportsShouldProcess,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[parameter(Mandatory)]
	[System.String]
	$Root,
	[parameter(Mandatory)]
	[System.String]
	$Issue,
	[parameter(Mandatory)]
	[System.String]
	$File)
	Begin {
	}
	Process {
	$path = Join-Path -Path $Root -ChildPath $Issue \| Join-Path -ChildPath "Attachments";
	New-Item -Path $path -Type Directory -Force \| Out-Null;
	$path = $path \| Join-Path -ChildPath ("~"+([string]::Join("~", ($File.Split("/") \| Select-Object -Last 3))));
	if(-Not (Test-Path -Path $path)) {
	Try {
	Invoke-WebRequest -OutFile $path -Uri $File -Headers @{ 'Authorization' = "Bearer $AccessToken" };
	} Catch {
	Write-Error -Message "Failed to get attachment: $File for issue: $Issue" -ErrorId 500 -TargetObject $Issue -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Add-DevopsWorkItem {
	[CmdletBinding(
	SupportsShouldProcess=$true,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Organization,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Project,
	[ValidateNotNullOrEmpty()]
	[System.String]
	[ValidateSet("Epic", "Feature", "User Story", "Bug", "Issue", "Task")]
	$Type,
	[System.Int32]
	$Id,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Data,
	[Switch]
	$Test)
	Begin {
	$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
	$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workitems/`$$($Type)?api-version=5.1&bypassRules=true&suppressNotifications=true";
	}
	Process {
	if ($PSCmdlet.ShouldProcess($Data)) {
	Try {
	if($Test) {
	Invoke-RestMethod -Method Patch -Uri $uri+"&validateOnly=true" -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
	} else {
	Invoke-RestMethod -Method Patch -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json-patch+json' -Body $Data
	}
	} Catch {
	Write-Error -Message "Failed to add issue for issue: $($Id)" -ErrorId 500 -TargetObject $Data -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Add-DevopsComment {
	[CmdletBinding(
	SupportsShouldProcess=$true,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Organization,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Project,
	[ValidateNotNullOrEmpty()]
	[System.Int32]
	$Id,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Data)
	Begin {
	$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)))
	$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/workItems/$Id/comments?api-version=5.1-preview.3";
	}
	Process {
	if ($PSCmdlet.ShouldProcess($Data)) {
	Try{
	Invoke-RestMethod -Method Post -Uri $uri -Headers @{ 'Authorization' = "Basic $basicAuth" } -ContentType 'application/json' -Body $Data
	} Catch {
	Write-Error -Message "Failed to add comments for issue: $($Id)" -ErrorId 500 -TargetObject $Id -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Add-DevopsAttachment {
	[CmdletBinding(
	SupportsShouldProcess=$true,
	ConfirmImpact="Medium"
	)]
	param (
	[parameter(Mandatory=$true)]
	[ValidateNotNullOrEmpty()]
	[System.String]
	$AccessToken,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Organization,
	[ValidateNotNullOrEmpty()]
	[System.String]
	$Project,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	[System.IO.FileInfo]
	$File)
	Begin {
	$basicAuth = [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes(("{0}:{1}" -f '', $AccessToken)));
	$uri = "https://dev.azure.com/$Organization/$Project/_apis/wit/attachments?api-version=5.0";
	}
	Process {
	if ($PSCmdlet.ShouldProcess($FileName)) {
	Try{
	$image = [System.IO.File]::ReadAllBytes($File.FullName);
	Invoke-RestMethod -Uri $uri+"&fileName=$([System.Web.HttpUtility]::UrlEncode($File.Name))" -Method Post -Headers @{ Authorization=("Basic {0}" -f $basicAuth) } -ContentType application/json -Body $image;
	} Catch {
	Write-Error -Message "Failed to add attachments for issue: $($File.FullName)" -ErrorId 500 -TargetObject $File -Category InvalidResult;
	}
	}
	}
	End {
	}
	}


	function Convert-IssueToWorkItem {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param (
	[parameter(Mandatory=$true)]
	$RootIterationPath,
	[parameter(Mandatory=$true,ValueFromPipeline)]
	$Issue
	)
	Begin {
	Import-Module MarkdownToHtml -Force;
	$userMap = @{
	#https://gitlab.com/api/v4/groups/mygroupid/members GitLabUserUserId="AzureDevOpsDisplayName",
	123 ="John Doe";
	};
	}
	Process {
	if([String]::IsNullOrWhiteSpace($Issue.description)) {
	$content = "`" `"";
	} else {
	$content = ((Convert-MarkdownToHTMLFragment -Markdown ($Issue.description \| Format-BadCharacters)).HtmlFragment).Replace("₹", "₹").Replace("'", "'") \| ConvertTo-Json;
	}
	$title = [System.Web.HttpUtility]::HtmlEncode(($Issue.title \| Format-BadCharacters));
	if($title.Length -gt 255){$title = $title.Substring(0,247);}
	$author = $($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.author.id}).Value;
	$assignee = ($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
	$closedby = ($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.closed_by.id}).Value;
	$tags = [String]::Join(";", $Issue.labels+$Issue.state);
	if($Issue.milestone) {
	$iteration = $RootIterationPath+"\\"+$Issue.milestone.title.Replace("?", "").Replace("&", "and");
	} else {
	$iteration = $RootIterationPath;
	}
	if($null -ne $Issue.assignee){
	$user = ($userMap.GetEnumerator() \| Where-Object{$_.Name -eq $Issue.assignee.id}).Value;
	$assignee = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.AssignedTo`",
	`"from`": null,
	`"value`": `"$($user)`"
	}"
	} else {
	$assignee = "";
	}
	$type = @("User Story", "Bug")[$Issue.labels.Contains("Bug")];
	if($type -eq "Bug") {
	$body = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/Microsoft.VSTS.TCM.ReproSteps`",
	`"from`": null,
	`"value`": $content
	}"
	} else {
	$body = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.Description`",
	`"from`": null,
	`"value`": $content
	}"
	}
	if($Issue.state -eq "closed"){
	$closed = ",
	{
	`"op`": `"add`",
	`"path`": `"/fields/Microsoft.VSTS.Common.ClosedBy`",
	`"from`": null,
	`"value`": `"$($closedby)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/Microsoft.VSTS.Common.ClosedDate`",
	`"from`": null,
	`"value`": `"$($Issue.closed_at)`"
	}";
	} else {
	$closed = "";
	}
	return "[
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.IterationPath`",
	`"from`": null,
	`"value`": `"$($iteration)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.Title`",
	`"from`": null,
	`"value`": `"$($Issue.iid): $($title)`"
	}$($body),
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.State`",
	`"from`": null,
	`"value`": `"$(@{"opened"="New"; "closed" = "New"}[$Issue.state])`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.Tags`",
	`"from`": null,
	`"value`": `"$($tags)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.CreatedBy`",
	`"from`": null,
	`"value`": `"$($author)`"
	},
	{
	`"op`": `"add`",
	`"path`": `"/fields/System.CreatedDate`",
	`"from`": null,
	`"value`": `"$($Issue.created_at)`"
	}$($assignee)$($closed)
	]";
	}
	End {
	}
	}

	function Convert-NoteToComment {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param(
	[parameter(Mandatory,ValueFromPipeline)]
	$Note
	)
	Begin {
	Import-Module MarkdownToHtml -Force;
	}
	Process {
	$header = "<p><b>$($Note.created_at.ToLocalTime().ToString('yyyy-MM-dd HH:mm:ss')) by $($Note.author.name)</b></p>";
	if([String]::IsNullOrWhiteSpace($Note.body)) {
	$content = "`" `"";
	} else {
	$content = ($header + (Convert-MarkdownToHTMLFragment -Markdown ($Note.body \| Format-BadCharacters)).HtmlFragment).Replace("₹", "₹") \| ConvertTo-Json;
	}
	return "{ `"text`": $content }";
	}
	End {
	}
	}

	function Format-BadCharacters {
	[System.Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseBOMForUnicodeEncodedFile', '', Scope='Function')]
	[CmdletBinding()]
	[OutputType([string])]
	param (
	[parameter(
	Mandatory,
	ValueFromPipeline,
	ValueFromPipelineByPropertyName,
	Position = 0)]
	[string] $Input
	)
	Process {
	return $Input.Replace("`“", "`"").Replace("`”", "`"").Replace(" ", " ").Replace("ä", "a").Replace("ë", "e");
	}
	}
	#Requires -PSEdition Core
	#Requires -Modules @{ ModuleName="GitLab"; ModuleVersion="0.0"; }, @{ ModuleName="AzureDevops"; ModuleVersion="0.0"; }, @{ ModuleName="GitLabDevOpsMigration"; ModuleVersion="0.0"; }, @{ ModuleName="MarkdownToHtml"; ModuleVersion="2.0.0"}

	throw "This is purely here so you dont accidentaly trigger this script, it is not designed to run end to end in one step, Please read the instructions"

	### Setup working directory
	$root = Join-Path -Path $(Get-Location) -ChildPath "MigrationWorkSpace"
	New-Item -Path $root -Type Directory -Force;

	### Step 1 - Download all GitLab Issues
	$issues = Get-GitlabIssues -AccessToken $GitLabAccessToken -ContainerPath $GitlabContainerPath;

	### Step 2 - Had to run this a few times so I always kept track of migrated ids in this file
	$MigratedIssueSucessPath = $root \| Join-Path -ChildPath "GtDIssueSucess.txt";
	if(Test-Path -Path $MigratedIssueSucessPath) {
	$migratedIssueIds = Get-Content $MigratedIssueSucessPath \| ForEach-Object{[int64]::Parse($_)}
	} else { $migratedIssueIds = @()}
	$issuesToMigrate = $issues \| Where-Object{ -not $migratedIssueIds.Contains($_.id)}


	### Step 3 - Save all issues into a folder
	$issuesToMigrate \| ForEach-Object {
	$folder = Join-Path -Path $root -ChildPath $_.id;
	New-Item -Path $folder -Type Directory -Force \| Out-Null;
	$_ \| ConvertTo-Json \| Out-File -FilePath (Join-Path -Path $folder -ChildPath issue.json) -Encoding utf8 -Force};

	### Step 4 - Get Unique Milestones and their status
	### These need to be created as sprints in devops manualy, I only had to create about a dozen so was easy enough
	$issuesToMigrate \| Select-Object -ExpandProperty milestone \| Select-Object title, state -Unique \| Sort-Object state, name

	### Step 5 - Download Comments
	$total = $issuesToMigrate.Count;
	$counter = 1;
	$issuesToMigrate \| Sort-Object id \| ForEach-Object -Begin {
	Write-Progress -Activity "Get-GitlabComments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	Write-Progress -Activity "Get-GitlabComments" -Status "$percent% Complete: $($_.id)" -PercentComplete $percent;
	$folder = Join-Path -Path $root -ChildPath $_.id;
	Get-GitlabComments -AccessToken $GitLabAccessToken -NotesLink $_._links.notes \|
	ConvertTo-Json \|
	Out-File -FilePath (Join-Path -Path $folder -ChildPath notes.json) -Encoding utf8 -Force;
	} -End {
	Write-Progress -Activity "Get-GitlabComments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}

	### Step 6 - Get all attachment links
	$files = @();
	#### Get attachment links from issues
	Get-ChildItem -Path $root \|
	ForEach-Object { $id = $_.Name;
	Get-ChildItem -Path $_.FullName -Filter issue.json \|
	Get-Content -Raw \|
	ConvertFrom-Json \|
	Where-Object { -not [String]::IsNullOrWhiteSpace($_.description) } \|
	ForEach-Object {
	$urlParts = $_.web_url.Split("/");
	$rootPath = [string]::Join("/", ($urlParts \| Select-Object -First ($urlParts.Count-2)));
	$_.description.Split("`n") \|
	ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} \|
	Where-Object { $_.Success } \|
	Select-Object -ExpandProperty Groups \|
	Where-Object { -Not $_.Value.StartsWith("]")} \|
	Select-Object -ExpandProperty Value \|
	ForEach-Object {
	$files+=New-Object PSObject -Property @{
	issue=$id;
	file=$rootPath+$_;}
	}
	}
	}

	#### Get attachment links from comments
	Get-ChildItem -Path $root \|
	ForEach-Object {
	$id = $_.Name;
	$urlParts = (Get-ChildItem -Path $_.FullName -Filter issue.json \|
	Get-Content -Raw \|
	ConvertFrom-Json).web_url.Split("/");
	$rootPath = [string]::Join("/", ($urlParts \| Select-Object -First ($urlParts.Count-2)));
	Get-ChildItem -Path $_.FullName -Filter notes.json \|
	Get-Content -Raw \|
	ConvertFrom-Json \|
	Where-Object { -not [String]::IsNullOrWhiteSpace($_.body) } \|
	ForEach-Object { $_.body.Split("`n") } \|
	ForEach-Object { [regex]::Match($_, "\]\((/uploads/.+)\)")} \|
	Where-Object { $_.Success } \|
	Select-Object -ExpandProperty Groups \|
	Where-Object { -Not $_.Value.StartsWith("]")} \|
	Select-Object -ExpandProperty Value \|
	ForEach-Object {
	$files+=New-Object PSObject -Property @{
	issue=$id;
	file=$rootPath+$_;} }
	}



	### Step 7 - Downlaod the attachments from gitlab NOTE: This can take while
	$total = $files.Count;
	$counter = 1;
	$files \| ForEach-Object -Begin {
	Write-Progress -Activity "Get-GitlabAttachments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	Write-Progress -Activity "Get-GitlabAttachments" -Status "$percent% Complete: $($_.issue)" -PercentComplete $percent;
	Get-GitLabAttachment -AccessToken $GitLabAccessToken -Root $root -Issue $_.Issue -File $_.File;
	} -End {
	Write-Progress -Activity "Get-GitlabAttachments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}

	### Step 8 - Upload and map attachments to offlined notes and comments
	$total = (Get-ChildItem -Path $root).Count;
	$counter = 1;
	Get-ChildItem -Path $root \|
	ForEach-Object -Begin {
	Write-Progress -Activity "Map-Attachments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	Write-Progress -Activity "Map-Attachments" -Status "$percent% Complete: $($_.Name)" -PercentComplete $percent;
	Push-Location -Path $_.FullName;
	if(Test-Path -Path Attachments) {
	$replace = Get-ChildItem -Path Attachments \| ForEach-Object {
	$name = $_ \| Select-Object -Property @{Name="Name";Expression={$_.Name.Replace("~","/")}} \| Select-Object -ExpandProperty Name;
	$upld = Add-DevopsAttachment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -File $_;
	Write-Verbose "$($name) $($upld.url)" -Verbose;
	@{Old=$name; New=$upld.url};
	}

	$replace \| ForEach-Object {
	$old = $_.Old;
	$new = $_.New;
	Get-ChildItem -File \| ForEach-Object {
	$fileName = $_.FullName;
	$content = Get-Content $_ \| ForEach-Object {
	$_.Replace($old, $new);
	};
	$content \| Out-File -FilePath $fileName -Encoding utf8 -Force;
	}
	}
	}
	Pop-Location;
	} -End {
	Write-Progress -Activity "Map-Attachments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}
	###
	# At this stage I had to do some manual fixes sometimes, important at this stage to re-load all the issues and commments from the filesystem as they have the updated data after uploading all the attachments
	###
	$issuesToMigrate = Get-ChildItem -Path $root -Filter issue.json -Recurse \| Get-Content -Raw \| ConvertFrom-Json

	### Step 9 - Issue + Comment Upload
	$failed = @();
	$total = $issuesToMigrate.Count;
	$test=$true; # This will allow you to validate before import, set to false for actual import
	$counter = 1;
	#### The Sort helped me judge how far into the process I was
	$issuesToMigrate \| Sort-Object id \|
	ForEach-Object -Begin {
	Write-Progress -Activity "Create Work Items and comments" -Status "0% Complete:" -PercentComplete 0;
	} -Process {
	$percent = $counter/$total*100;
	$counter++;
	$id = $_.id;
	### Note the new bug template needs to map the content to another field other than description, think it was called reposteps or similar
	$type = @("User Story", "Bug")[$_.labels.Contains("Bug")];
	Write-Progress -Activity "Create Work Items and comments" -Status "$percent% Complete: $($id)" -PercentComplete $percent;
	$_ \| Convert-IssueToWorkItem -RootIterationPath "Squad.Umenit" \| ForEach-Object{
	$workItem = $null;
	$workItem = Add-DevopsWorkItem -Test:$test -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $id -Type $type -Data $_;
	if($null -eq $workItem){
	$failed += @{$id=$_}
	} else {
	if(-not $test) {
	$comments = Get-Content -Path ($root \| Join-Path -ChildPath $id \| Join-Path -ChildPath "notes.json") -Raw \| ConvertFrom-Json \| Sort-Object -Property created_at;
	$comments.GetEnumerator() \| Convert-NoteToComment \| Add-DevopsComment -AccessToken $DevOpsAccessToken -Organization $DevOpsOrganization -Project $DevOpsProjectId -Id $workItem.id \| Out-Null;
	}
	}
	}
	} -End {
	Write-Progress -Activity "Create Work Items and comments" -Status "100% Complete:" -PercentComplete 100 -Completed
	}