From bfeb888c553d4c9f40e4cde5ac49cc34adf7bf0d Mon Sep 17 00:00:00 2001 From: dzaitsev Date: Wed, 7 May 2025 19:59:57 +0300 Subject: [PATCH] pipelines zip and gdrive upload updated --- build/jenkins/GDriveUpload | 137 ++++++++++++++----------------------- 1 file changed, 50 insertions(+), 87 deletions(-) diff --git a/build/jenkins/GDriveUpload b/build/jenkins/GDriveUpload index b43f2a3..c726cb7 100644 --- a/build/jenkins/GDriveUpload +++ b/build/jenkins/GDriveUpload @@ -10,8 +10,9 @@ pipeline { } environment { - GOOGLE_DRIVE_FOLDER = 'AzaionSuiteBuilds' // Hardcoded folder on Google Drive - RCLONE_CONFIG = 'C:/Program Files/rclone/rclone.conf' + GOOGLE_DRIVE_FOLDER = 'AzaionSuiteBuilds' + RCLONE_CONFIG = 'C:/Program Files/rclone/rclone.conf' + LATEST_ZIP_FILENAME = '' // Will be set in Find Latest Zip } stages { @@ -30,36 +31,29 @@ pipeline { stage('Find Latest Zip') { steps { script { - echo "Using build path: ${params.buildPath}" // Debug output for build path - echo "Finding latest zip file in: ${params.buildPath}" - def latestZipFilename = '' + echo "Using build path: ${params.buildPath}" + def latestZip = '' dir("${params.buildPath}") { def output = powershell(returnStdout: true, script: ''' $pattern = "AzaionSuite*.zip" $zipFiles = Get-ChildItem -Filter $pattern | Sort-Object Name -Descending if ($zipFiles.Count -gt 0) { - $latestZip = $zipFiles[0].Name - Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$latestZip" + Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$($zipFiles[0].Name)" } else { Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=" } ''').trim() - // Debug output to verify the response from the powershell script echo "PowerShell output: ${output}" - - def match = output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/ - if (!match || !match[0][1]?.trim()) { - echo "⚠️ No ZIP files matching the pattern were found. Skipping upload and cleanup stages." + def m = (output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/) + if (m && m[0][1]) { + latestZip = m[0][1] + echo "✅ Latest zip file selected: ${latestZip}" } else { - latestZipFilename = match[0][1] - echo "✅ Latest zip file selected: ${latestZipFilename}" + echo "⚠️ No zip found, skipping downstream stages." } } - - // Pass the filename to the next stages directly - env.LATEST_ZIP_FILENAME = latestZipFilename - echo "LATEST_ZIP_FILENAME after processing: ${env.LATEST_ZIP_FILENAME}" + env.LATEST_ZIP_FILENAME = latestZip } } } @@ -67,36 +61,32 @@ pipeline { stage('Upload If Not Exists & Always Remove Local') { when { expression { - echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" // Debug output + echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" return env.LATEST_ZIP_FILENAME?.trim() } } steps { - echo "Checking Google Drive for existing ZIP and uploading if needed..." + echo "Uploading ${env.LATEST_ZIP_FILENAME} if needed..." powershell """ - \$fileName = "${env.LATEST_ZIP_FILENAME}" - \$folder = "${GOOGLE_DRIVE_FOLDER}" + \$fileName = "${env.LATEST_ZIP_FILENAME}" + \$folder = "${GOOGLE_DRIVE_FOLDER}" \$rcloneRemote = "AzaionGoogleDrive:\$folder" - \$localFilePath = "${params.buildPath}/\$fileName" + \$localPath = "${params.buildPath}/\$fileName" - Write-Output "Checking for existing files in: \$rcloneRemote" - \$existingFiles = rclone lsf --files-only \$rcloneRemote - Write-Output "Existing files:" - Write-Output \$existingFiles + Write-Output "Existing on drive:" + \$existing = rclone lsf --files-only \$rcloneRemote + Write-Output \$existing - if (\$existingFiles -match "^\$fileName\$") { - Write-Output "File '\$fileName' already exists on Google Drive. Skipping upload." + if (\$existing -match "^\$fileName\$") { + Write-Output "Already exists; skipping upload." } else { - Write-Output "Uploading '\$fileName' to Google Drive..." - rclone copy "\$localFilePath" \$rcloneRemote --progress --drive-chunk-size 64M - Write-Output "Upload complete." + Write-Output "Uploading..." + rclone copy \$localPath \$rcloneRemote --drive-chunk-size 64M --progress } - if (Test-Path \$localFilePath) { - Remove-Item -Force \$localFilePath - Write-Output "Local file deleted: \$localFilePath" - } else { - Write-Output "Local file already deleted or not found." + if (Test-Path \$localPath) { + Remove-Item -Force \$localPath + Write-Output "Local deleted." } """ } @@ -105,59 +95,36 @@ pipeline { stage('Cleanup Older Files on Google Drive') { when { expression { - echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" // Debug output + echo "LATEST_ZIP_FILENAME is: '${env.LATEST_ZIP_FILENAME}'" return env.LATEST_ZIP_FILENAME?.trim() } } steps { - echo "Cleaning up older files on Google Drive..." + echo "Cleaning up older files..." powershell """ - Write-Output "Listing all files in the folder ${GOOGLE_DRIVE_FOLDER} on Google Drive..." - \$files = rclone lsf --files-only AzaionGoogleDrive:${GOOGLE_DRIVE_FOLDER} - Write-Output "Files found on Google Drive:" - Write-Output \$files + \$folder = "${GOOGLE_DRIVE_FOLDER}" + \$files = rclone lsf --files-only AzaionGoogleDrive:\$folder | Where-Object { \$_ -match '^AzaionSuite.*\\.zip\$' } - # Extract date-time from filenames and sort them - \$filesArray = \$files -split "`n" | Where-Object { \$_ -ne "" } - \$filesWithDateTime = @() - - foreach (\$file in \$filesArray) { - if (\$file -match "AzaionSuite.Iterative.(\d{8})-(\d{6}).zip") { - \$date = [datetime]::ParseExact(\$matches[1], "yyyyMMdd", $null) - \$time = [datetime]::ParseExact(\$matches[2], "HHmmss", $null) - \$dateTime = \$date.Add(\$time.TimeOfDay) - \$filesWithDateTime += [PSCustomObject]@{ - FileName = \$file - DateTime = \$dateTime - } + # Build an array of objects with parsed DateTime + \$meta = foreach (\$f in \$files) { + if (\$f -match 'AzaionSuite\\.Iterative\\.(\\d{8})-(\\d{6})\\.zip') { + \$dt = [datetime]::ParseExact(\$matches[1] + \$matches[2], 'yyyyMMddHHmmss', \$null) + } elseif (\$f -match 'AzaionSuite1\\.\\d+\\.\\d+\\.\\d+-(\\d{8})-(\\d{6})\\.zip') { + \$dt = [datetime]::ParseExact(\$matches[1] + \$matches[2], 'yyyyMMddHHmmss', \$null) + } else { + # if it doesn't match expected formats, skip + continue } + [PSCustomObject]@{ Name = \$f; Date = \$dt } } - # Sort files based on DateTime in descending order - \$sortedFiles = \$filesWithDateTime | Sort-Object DateTime -Descending - Write-Output "Sorted files (newest to oldest):" - \$sortedFiles | ForEach-Object { Write-Output "\$($_.FileName)" } - - # Keep the latest 3 files and delete the rest - \$filesToDelete = \$sortedFiles | Select-Object -Skip 3 - Write-Output "Files to delete (older than 3 latest):" - \$filesToDelete | ForEach-Object { Write-Output "\$($_.FileName)" } - - if (\$filesToDelete.Count -gt 0) { - \$tempFile = [System.IO.Path]::GetTempFileName() - \$filesToDelete.FileName | Set-Content -Path \$tempFile -Encoding utf8 - - Write-Output "Contents of temporary delete list file (\$tempFile):" - Get-Content \$tempFile - - foreach (\$file in \$filesToDelete.FileName) { - Write-Output "Deleting \$file..." - rclone deletefile AzaionGoogleDrive:${GOOGLE_DRIVE_FOLDER}/\$file - } - - Remove-Item -Path \$tempFile + # Sort by Date descending, skip the first 3 (newest), delete the rest + \$toDelete = \$meta | Sort-Object Date -Descending | Select-Object -Skip 3 + if (\$toDelete) { + Write-Output "Deleting:\" + \$toDelete | ForEach-Object { Write-Output \$_ .Name; rclone deletefile AzaionGoogleDrive:\$folder/\$($_.Name) } } else { - Write-Output "No files to delete." + Write-Output "Nothing to delete." } """ } @@ -166,17 +133,13 @@ pipeline { post { always { - echo 'Executing post-build cleanup...' + echo 'Cleaning up temp files...' powershell ''' - $uploadDir = "temp_upload" - if (Test-Path $uploadDir) { - Remove-Item -Recurse -Force $uploadDir - } + if (Test-Path temp_upload) { Remove-Item -Recurse -Force temp_upload } ''' } - failure { - echo 'Pipeline failed. Check logs for details.' + echo 'Pipeline failed; check the logs above.' } } }