mirror of
https://github.com/azaion/annotations.git
synced 2026-04-22 11:06:30 +00:00
pipelines zip and gdrive upload
updated
This commit is contained in:
dzaitsev
+50
-87
@@ -10,8 +10,9 @@ pipeline {
|
|||||||
}
|
}
|
||||||
|
|
||||||
environment {
|
environment {
|
||||||
GOOGLE_DRIVE_FOLDER = 'AzaionSuiteBuilds' // Hardcoded folder on Google Drive
|
GOOGLE_DRIVE_FOLDER = 'AzaionSuiteBuilds'
|
||||||
RCLONE_CONFIG = 'C:/Program Files/rclone/rclone.conf'
|
RCLONE_CONFIG = 'C:/Program Files/rclone/rclone.conf'
|
||||||
|
LATEST_ZIP_FILENAME = '' // Will be set in Find Latest Zip
|
||||||
}
|
}
|
||||||
|
|
||||||
stages {
|
stages {
|
||||||
@@ -30,36 +31,29 @@ pipeline {
|
|||||||
stage('Find Latest Zip') {
|
stage('Find Latest Zip') {
|
||||||
steps {
|
steps {
|
||||||
script {
|
script {
|
||||||
echo "Using build path: ${params.buildPath}" // Debug output for build path
|
echo "Using build path: ${params.buildPath}"
|
||||||
echo "Finding latest zip file in: ${params.buildPath}"
|
def latestZip = ''
|
||||||
def latestZipFilename = ''
|
|
||||||
dir("${params.buildPath}") {
|
dir("${params.buildPath}") {
|
||||||
def output = powershell(returnStdout: true, script: '''
|
def output = powershell(returnStdout: true, script: '''
|
||||||
$pattern = "AzaionSuite*.zip"
|
$pattern = "AzaionSuite*.zip"
|
||||||
$zipFiles = Get-ChildItem -Filter $pattern | Sort-Object Name -Descending
|
$zipFiles = Get-ChildItem -Filter $pattern | Sort-Object Name -Descending
|
||||||
if ($zipFiles.Count -gt 0) {
|
if ($zipFiles.Count -gt 0) {
|
||||||
$latestZip = $zipFiles[0].Name
|
Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$($zipFiles[0].Name)"
|
||||||
Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$latestZip"
|
|
||||||
} else {
|
} else {
|
||||||
Write-Output "::SET-ENV::LATEST_ZIP_FILENAME="
|
Write-Output "::SET-ENV::LATEST_ZIP_FILENAME="
|
||||||
}
|
}
|
||||||
''').trim()
|
''').trim()
|
||||||
|
|
||||||
// Debug output to verify the response from the powershell script
|
|
||||||
echo "PowerShell output: ${output}"
|
echo "PowerShell output: ${output}"
|
||||||
|
def m = (output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/)
|
||||||
def match = output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/
|
if (m && m[0][1]) {
|
||||||
if (!match || !match[0][1]?.trim()) {
|
latestZip = m[0][1]
|
||||||
echo "⚠️ No ZIP files matching the pattern were found. Skipping upload and cleanup stages."
|
echo "✅ Latest zip file selected: ${latestZip}"
|
||||||
} else {
|
} else {
|
||||||
latestZipFilename = match[0][1]
|
echo "⚠️ No zip found, skipping downstream stages."
|
||||||
echo "✅ Latest zip file selected: ${latestZipFilename}"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
env.LATEST_ZIP_FILENAME = latestZip
|
||||||
// Pass the filename to the next stages directly
|
|
||||||
env.LATEST_ZIP_FILENAME = latestZipFilename
|
|
||||||
echo "LATEST_ZIP_FILENAME after processing: ${env.LATEST_ZIP_FILENAME}"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -67,36 +61,32 @@ pipeline {
|
|||||||
stage('Upload If Not Exists & Always Remove Local') {
|
stage('Upload If Not Exists & Always Remove Local') {
|
||||||
when {
|
when {
|
||||||
expression {
|
expression {
|
||||||
echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" // Debug output
|
echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'"
|
||||||
return env.LATEST_ZIP_FILENAME?.trim()
|
return env.LATEST_ZIP_FILENAME?.trim()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
steps {
|
steps {
|
||||||
echo "Checking Google Drive for existing ZIP and uploading if needed..."
|
echo "Uploading ${env.LATEST_ZIP_FILENAME} if needed..."
|
||||||
powershell """
|
powershell """
|
||||||
\$fileName = "${env.LATEST_ZIP_FILENAME}"
|
\$fileName = "${env.LATEST_ZIP_FILENAME}"
|
||||||
\$folder = "${GOOGLE_DRIVE_FOLDER}"
|
\$folder = "${GOOGLE_DRIVE_FOLDER}"
|
||||||
\$rcloneRemote = "AzaionGoogleDrive:\$folder"
|
\$rcloneRemote = "AzaionGoogleDrive:\$folder"
|
||||||
\$localFilePath = "${params.buildPath}/\$fileName"
|
\$localPath = "${params.buildPath}/\$fileName"
|
||||||
|
|
||||||
Write-Output "Checking for existing files in: \$rcloneRemote"
|
Write-Output "Existing on drive:"
|
||||||
\$existingFiles = rclone lsf --files-only \$rcloneRemote
|
\$existing = rclone lsf --files-only \$rcloneRemote
|
||||||
Write-Output "Existing files:"
|
Write-Output \$existing
|
||||||
Write-Output \$existingFiles
|
|
||||||
|
|
||||||
if (\$existingFiles -match "^\$fileName\$") {
|
if (\$existing -match "^\$fileName\$") {
|
||||||
Write-Output "File '\$fileName' already exists on Google Drive. Skipping upload."
|
Write-Output "Already exists; skipping upload."
|
||||||
} else {
|
} else {
|
||||||
Write-Output "Uploading '\$fileName' to Google Drive..."
|
Write-Output "Uploading..."
|
||||||
rclone copy "\$localFilePath" \$rcloneRemote --progress --drive-chunk-size 64M
|
rclone copy \$localPath \$rcloneRemote --drive-chunk-size 64M --progress
|
||||||
Write-Output "Upload complete."
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (Test-Path \$localFilePath) {
|
if (Test-Path \$localPath) {
|
||||||
Remove-Item -Force \$localFilePath
|
Remove-Item -Force \$localPath
|
||||||
Write-Output "Local file deleted: \$localFilePath"
|
Write-Output "Local deleted."
|
||||||
} else {
|
|
||||||
Write-Output "Local file already deleted or not found."
|
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
@@ -105,59 +95,36 @@ pipeline {
|
|||||||
stage('Cleanup Older Files on Google Drive') {
|
stage('Cleanup Older Files on Google Drive') {
|
||||||
when {
|
when {
|
||||||
expression {
|
expression {
|
||||||
echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" // Debug output
|
echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'"
|
||||||
return env.LATEST_ZIP_FILENAME?.trim()
|
return env.LATEST_ZIP_FILENAME?.trim()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
steps {
|
steps {
|
||||||
echo "Cleaning up older files on Google Drive..."
|
echo "Cleaning up older files..."
|
||||||
powershell """
|
powershell """
|
||||||
Write-Output "Listing all files in the folder ${GOOGLE_DRIVE_FOLDER} on Google Drive..."
|
\$folder = "${GOOGLE_DRIVE_FOLDER}"
|
||||||
\$files = rclone lsf --files-only AzaionGoogleDrive:${GOOGLE_DRIVE_FOLDER}
|
\$files = rclone lsf --files-only AzaionGoogleDrive:\$folder | Where-Object { \$_ -match '^AzaionSuite.*\\.zip\$' }
|
||||||
Write-Output "Files found on Google Drive:"
|
|
||||||
Write-Output \$files
|
|
||||||
|
|
||||||
# Extract date-time from filenames and sort them
|
# Build an array of objects with parsed DateTime
|
||||||
\$filesArray = \$files -split "`n" | Where-Object { \$_ -ne "" }
|
\$meta = foreach (\$f in \$files) {
|
||||||
\$filesWithDateTime = @()
|
if (\$f -match 'AzaionSuite\\.Iterative\\.(\\d{8})-(\\d{6})\\.zip') {
|
||||||
|
\$dt = [datetime]::ParseExact(\$matches[1] + \$matches[2], 'yyyyMMddHHmmss', \$null)
|
||||||
foreach (\$file in \$filesArray) {
|
} elseif (\$f -match 'AzaionSuite1\\.\\d+\\.\\d+\\.\\d+-(\\d{8})-(\\d{6})\\.zip') {
|
||||||
if (\$file -match "AzaionSuite.Iterative.(\d{8})-(\d{6}).zip") {
|
\$dt = [datetime]::ParseExact(\$matches[1] + \$matches[2], 'yyyyMMddHHmmss', \$null)
|
||||||
\$date = [datetime]::ParseExact(\$matches[1], "yyyyMMdd", $null)
|
|
||||||
\$time = [datetime]::ParseExact(\$matches[2], "HHmmss", $null)
|
|
||||||
\$dateTime = \$date.Add(\$time.TimeOfDay)
|
|
||||||
\$filesWithDateTime += [PSCustomObject]@{
|
|
||||||
FileName = \$file
|
|
||||||
DateTime = \$dateTime
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# Sort files based on DateTime in descending order
|
|
||||||
\$sortedFiles = \$filesWithDateTime | Sort-Object DateTime -Descending
|
|
||||||
Write-Output "Sorted files (newest to oldest):"
|
|
||||||
\$sortedFiles | ForEach-Object { Write-Output "\$($_.FileName)" }
|
|
||||||
|
|
||||||
# Keep the latest 3 files and delete the rest
|
|
||||||
\$filesToDelete = \$sortedFiles | Select-Object -Skip 3
|
|
||||||
Write-Output "Files to delete (older than 3 latest):"
|
|
||||||
\$filesToDelete | ForEach-Object { Write-Output "\$($_.FileName)" }
|
|
||||||
|
|
||||||
if (\$filesToDelete.Count -gt 0) {
|
|
||||||
\$tempFile = [System.IO.Path]::GetTempFileName()
|
|
||||||
\$filesToDelete.FileName | Set-Content -Path \$tempFile -Encoding utf8
|
|
||||||
|
|
||||||
Write-Output "Contents of temporary delete list file (\$tempFile):"
|
|
||||||
Get-Content \$tempFile
|
|
||||||
|
|
||||||
foreach (\$file in \$filesToDelete.FileName) {
|
|
||||||
Write-Output "Deleting \$file..."
|
|
||||||
rclone deletefile AzaionGoogleDrive:${GOOGLE_DRIVE_FOLDER}/\$file
|
|
||||||
}
|
|
||||||
|
|
||||||
Remove-Item -Path \$tempFile
|
|
||||||
} else {
|
} else {
|
||||||
Write-Output "No files to delete."
|
# if it doesn't match expected formats, skip
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
[PSCustomObject]@{ Name = \$f; Date = \$dt }
|
||||||
|
}
|
||||||
|
|
||||||
|
# Sort by Date descending, skip the first 3 (newest), delete the rest
|
||||||
|
\$toDelete = \$meta | Sort-Object Date -Descending | Select-Object -Skip 3
|
||||||
|
if (\$toDelete) {
|
||||||
|
Write-Output "Deleting:\"
|
||||||
|
\$toDelete | ForEach-Object { Write-Output \$_ .Name; rclone deletefile AzaionGoogleDrive:\$folder/\$($_.Name) }
|
||||||
|
} else {
|
||||||
|
Write-Output "Nothing to delete."
|
||||||
}
|
}
|
||||||
"""
|
"""
|
||||||
}
|
}
|
||||||
@@ -166,17 +133,13 @@ pipeline {
|
|||||||
|
|
||||||
post {
|
post {
|
||||||
always {
|
always {
|
||||||
echo 'Executing post-build cleanup...'
|
echo 'Cleaning up temp files...'
|
||||||
powershell '''
|
powershell '''
|
||||||
$uploadDir = "temp_upload"
|
if (Test-Path temp_upload) { Remove-Item -Recurse -Force temp_upload }
|
||||||
if (Test-Path $uploadDir) {
|
|
||||||
Remove-Item -Recurse -Force $uploadDir
|
|
||||||
}
|
|
||||||
'''
|
'''
|
||||||
}
|
}
|
||||||
|
|
||||||
failure {
|
failure {
|
||||||
echo 'Pipeline failed. Check logs for details.'
|
echo 'Pipeline failed; check the logs above.'
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user