From 16c264ec9a1b0f6c7cad16210f08536dde984fb7 Mon Sep 17 00:00:00 2001 From: dzaitsev Date: Wed, 7 May 2025 20:11:54 +0300 Subject: [PATCH] pipelines zip and gdrive upload updated --- build/jenkins/GDriveUpload | 140 +++++++++++++++++++++++-------------- 1 file changed, 88 insertions(+), 52 deletions(-) diff --git a/build/jenkins/GDriveUpload b/build/jenkins/GDriveUpload index d08c663..b43f2a3 100644 --- a/build/jenkins/GDriveUpload +++ b/build/jenkins/GDriveUpload @@ -10,9 +10,8 @@ pipeline { } environment { - GOOGLE_DRIVE_FOLDER = 'AzaionSuiteBuilds' - RCLONE_CONFIG = 'C:/Program Files/rclone/rclone.conf' - LATEST_ZIP_FILENAME = '' // Initial value + GOOGLE_DRIVE_FOLDER = 'AzaionSuiteBuilds' // Hardcoded folder on Google Drive + RCLONE_CONFIG = 'C:/Program Files/rclone/rclone.conf' } stages { @@ -31,32 +30,36 @@ pipeline { stage('Find Latest Zip') { steps { script { - echo "Using build path: ${params.buildPath}" - def latestZip = '' + echo "Using build path: ${params.buildPath}" // Debug output for build path + echo "Finding latest zip file in: ${params.buildPath}" + def latestZipFilename = '' dir("${params.buildPath}") { def output = powershell(returnStdout: true, script: ''' $pattern = "AzaionSuite*.zip" $zipFiles = Get-ChildItem -Filter $pattern | Sort-Object Name -Descending if ($zipFiles.Count -gt 0) { - Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$($zipFiles[0].Name)" + $latestZip = $zipFiles[0].Name + Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$latestZip" } else { Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=" } ''').trim() + // Debug output to verify the response from the powershell script echo "PowerShell output: ${output}" - def match = (output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/) - if (match) { - latestZip = match[0][1] - echo "✅ Latest zip file selected: ${latestZip}" + def match = output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/ + if (!match || !match[0][1]?.trim()) { + echo "⚠️ No ZIP files matching the pattern were found. Skipping upload and cleanup stages." } else { - echo "⚠️ No zip found." - latestZip = 'none' // Handle case where no file is found + latestZipFilename = match[0][1] + echo "✅ Latest zip file selected: ${latestZipFilename}" } } - // Directly assign the value to the env variable for use in other stages - env.LATEST_ZIP_FILENAME = latestZip + + // Pass the filename to the next stages directly + env.LATEST_ZIP_FILENAME = latestZipFilename + echo "LATEST_ZIP_FILENAME after processing: ${env.LATEST_ZIP_FILENAME}" } } } @@ -64,31 +67,36 @@ pipeline { stage('Upload If Not Exists & Always Remove Local') { when { expression { - return env.LATEST_ZIP_FILENAME != 'none' + echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" // Debug output + return env.LATEST_ZIP_FILENAME?.trim() } } steps { - echo "Uploading ${env.LATEST_ZIP_FILENAME} if needed..." + echo "Checking Google Drive for existing ZIP and uploading if needed..." powershell """ - \$fileName = "${env.LATEST_ZIP_FILENAME}" - \$folder = "${GOOGLE_DRIVE_FOLDER}" + \$fileName = "${env.LATEST_ZIP_FILENAME}" + \$folder = "${GOOGLE_DRIVE_FOLDER}" \$rcloneRemote = "AzaionGoogleDrive:\$folder" - \$localPath = "${params.buildPath}/\$fileName" + \$localFilePath = "${params.buildPath}/\$fileName" - Write-Output "Existing on drive:" - \$existing = rclone lsf --files-only \$rcloneRemote - Write-Output \$existing + Write-Output "Checking for existing files in: \$rcloneRemote" + \$existingFiles = rclone lsf --files-only \$rcloneRemote + Write-Output "Existing files:" + Write-Output \$existingFiles - if (\$existing -match "^\$fileName\$") { - Write-Output "Already exists; skipping upload." + if (\$existingFiles -match "^\$fileName\$") { + Write-Output "File '\$fileName' already exists on Google Drive. Skipping upload." } else { - Write-Output "Uploading..." - rclone copy \$localPath \$rcloneRemote --drive-chunk-size 64M --progress + Write-Output "Uploading '\$fileName' to Google Drive..." + rclone copy "\$localFilePath" \$rcloneRemote --progress --drive-chunk-size 64M + Write-Output "Upload complete." } - if (Test-Path \$localPath) { - Remove-Item -Force \$localPath - Write-Output "Local deleted." + if (Test-Path \$localFilePath) { + Remove-Item -Force \$localFilePath + Write-Output "Local file deleted: \$localFilePath" + } else { + Write-Output "Local file already deleted or not found." } """ } @@ -97,35 +105,59 @@ pipeline { stage('Cleanup Older Files on Google Drive') { when { expression { - return env.LATEST_ZIP_FILENAME != 'none' + echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" // Debug output + return env.LATEST_ZIP_FILENAME?.trim() } } steps { - echo "Cleaning up older files..." + echo "Cleaning up older files on Google Drive..." powershell """ - \$folder = "${GOOGLE_DRIVE_FOLDER}" - \$files = rclone lsf --files-only AzaionGoogleDrive:\$folder | Where-Object { \$_ -match '^AzaionSuite.*\\.zip\$' } + Write-Output "Listing all files in the folder ${GOOGLE_DRIVE_FOLDER} on Google Drive..." + \$files = rclone lsf --files-only AzaionGoogleDrive:${GOOGLE_DRIVE_FOLDER} + Write-Output "Files found on Google Drive:" + Write-Output \$files - # Build an array of objects with parsed DateTime - \$meta = foreach (\$f in \$files) { - if (\$f -match 'AzaionSuite\\.Iterative\\.(\\d{8})-(\\d{6})\\.zip') { - \$dt = [datetime]::ParseExact(\$matches[1] + \$matches[2], 'yyyyMMddHHmmss', \$null) - } elseif (\$f -match 'AzaionSuite1\\.\\d+\\.\\d+\\.\\d+-(\\d{8})-(\\d{6})\\.zip') { - \$dt = [datetime]::ParseExact(\$matches[1] + \$matches[2], 'yyyyMMddHHmmss', \$null) - } else { - # if it doesn't match expected formats, skip - continue + # Extract date-time from filenames and sort them + \$filesArray = \$files -split "`n" | Where-Object { \$_ -ne "" } + \$filesWithDateTime = @() + + foreach (\$file in \$filesArray) { + if (\$file -match "AzaionSuite.Iterative.(\d{8})-(\d{6}).zip") { + \$date = [datetime]::ParseExact(\$matches[1], "yyyyMMdd", $null) + \$time = [datetime]::ParseExact(\$matches[2], "HHmmss", $null) + \$dateTime = \$date.Add(\$time.TimeOfDay) + \$filesWithDateTime += [PSCustomObject]@{ + FileName = \$file + DateTime = \$dateTime + } } - [PSCustomObject]@{ Name = \$f; Date = \$dt } } - # Sort by Date descending, skip the first 3 (newest), delete the rest - \$toDelete = \$meta | Sort-Object Date -Descending | Select-Object -Skip 3 - if (\$toDelete) { - Write-Output "Deleting:\" - \$toDelete | ForEach-Object { Write-Output \$_ .Name; rclone deletefile AzaionGoogleDrive:\$folder/\$($_.Name) } + # Sort files based on DateTime in descending order + \$sortedFiles = \$filesWithDateTime | Sort-Object DateTime -Descending + Write-Output "Sorted files (newest to oldest):" + \$sortedFiles | ForEach-Object { Write-Output "\$($_.FileName)" } + + # Keep the latest 3 files and delete the rest + \$filesToDelete = \$sortedFiles | Select-Object -Skip 3 + Write-Output "Files to delete (older than 3 latest):" + \$filesToDelete | ForEach-Object { Write-Output "\$($_.FileName)" } + + if (\$filesToDelete.Count -gt 0) { + \$tempFile = [System.IO.Path]::GetTempFileName() + \$filesToDelete.FileName | Set-Content -Path \$tempFile -Encoding utf8 + + Write-Output "Contents of temporary delete list file (\$tempFile):" + Get-Content \$tempFile + + foreach (\$file in \$filesToDelete.FileName) { + Write-Output "Deleting \$file..." + rclone deletefile AzaionGoogleDrive:${GOOGLE_DRIVE_FOLDER}/\$file + } + + Remove-Item -Path \$tempFile } else { - Write-Output "Nothing to delete." + Write-Output "No files to delete." } """ } @@ -134,13 +166,17 @@ pipeline { post { always { - echo 'Cleaning up temp files...' + echo 'Executing post-build cleanup...' powershell ''' - if (Test-Path temp_upload) { Remove-Item -Recurse -Force temp_upload } + $uploadDir = "temp_upload" + if (Test-Path $uploadDir) { + Remove-Item -Recurse -Force $uploadDir + } ''' } + failure { - echo 'Pipeline failed; check the logs above.' + echo 'Pipeline failed. Check logs for details.' } } }