From 0c0cc1bb839da7d1db16c195b7725b03c6e1a3ac Mon Sep 17 00:00:00 2001 From: dzaitsev Date: Sun, 4 May 2025 14:25:28 +0300 Subject: [PATCH] gdrive update --- build/jenkins/GDriveUpload | 314 +++++++------------------------------ 1 file changed, 53 insertions(+), 261 deletions(-) diff --git a/build/jenkins/GDriveUpload b/build/jenkins/GDriveUpload index 8ad4a11..b72410c 100644 --- a/build/jenkins/GDriveUpload +++ b/build/jenkins/GDriveUpload @@ -1,283 +1,75 @@ -// Jenkins Pipeline script to find the latest zip file directly in the artifacts directory -// and upload it to Google Drive with retention. -// Retention is based on the timestamp embedded in the filename and performed entirely in PowerShell. -// Designed to run on a Windows agent, specifically on the machine where the main build creates artifacts. -// Assumes rclone is installed and its directory is added to the system's PATH on the Windows agent. - pipeline { - // Agent should be the SAME Windows VM agent where the main build job runs - // and creates the artifacts in the MAIN_BUILD_ARTIFACTS_DIR. - agent { label 'Win10-BuildMachine' } // Replace with your Windows agent label if different - - environment { - // Google Drive settings - GDRIVE_REMOTE = 'AzaionGoogleDrive:AzaionSuiteBuilds' // Your rclone remote name and path - // Use a relative path within the workspace for the temporary directory on Windows - // This temporary directory is still useful for rclone's internal operations and cleanup - TMP_UPLOAD_DIR = 'temp_upload' - // Path to rclone.conf on the Windows agent. - // Adjust this path if your rclone.conf is located elsewhere on the Windows VM. - // Using a relative path from the workspace root is often best practice. - RCLONE_CONFIG = 'rclone.conf' // Assuming rclone.conf is in the workspace root - - // Number of latest files to keep on Google Drive - FILES_TO_KEEP = 3 - - // Define the FULL path to the directory containing the zip files. - // This job will operate directly within this directory. - // ** IMPORTANT: Replace 'C:/Jenkins/workspace/AzaionSuite/suite' with the actual path ** - TARGET_ARTIFACTS_DIR = 'C:/Jenkins/workspace/AzaionSuite/suite' // <<== UPDATE THIS - - // Define the name of the latest zip file found as an environment variable - // This makes it easier to reference in later stages - LATEST_ZIP_FILENAME = '' // This will be set dynamically in the 'Find Latest Zip and Upload' stage - } + agent { label 'windows' } stages { stage('Initialize Workspace') { steps { - echo "Initializing workspace on Windows agent..." - // Use standard Windows PowerShell for directory creation and cleanup - // Ensure paths are quoted for safety and use backslashes for Windows paths - powershell """ - # Create temporary upload directory within THIS job's workspace - New-Item -ItemType Directory -Force -Path "${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}" - - # Clean up previous temporary files in the upload directory - Remove-Item -Recurse -Force "${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}\\\\*" -ErrorAction SilentlyContinue - """ + echo 'Initializing workspace on Windows agent...' + powershell ''' + $uploadDir = "temp_upload" + if (-Not (Test-Path $uploadDir)) { + New-Item -ItemType Directory -Path $uploadDir | Out-Null + } + ''' } } - stage('Find Latest Zip and Upload') { // Renamed stage + stage('Find Latest Zip and Upload') { steps { - script { // Wrap steps in a script block + script { echo "Starting 'Find Latest Zip and Upload' stage." - - // Change directory to the target artifacts folder where the zip files are located - dir("${env.TARGET_ARTIFACTS_DIR}") { - echo "Operating in directory: ${pwd()}" - - // Use PowerShell to find the latest zip file based on filename timestamp - // This logic is similar to the 'Archive Latest Zip' stage from the combined pipeline - def powershellOutput = powershell(script: """ - \$zipPattern = "AzaionSuite*-*-*.zip" # Pattern for zip files - Write-Host "Searching for latest zip file matching '\$zipPattern' in '\$PWD'..." - - # Find all zip files matching the pattern - # Use -ErrorAction Stop to fail if Get-ChildItem fails - \$zipFiles = Get-ChildItem -Path . -Filter \$zipPattern -ErrorAction Stop - - if (\$zipFiles.Count -eq 0) { - Write-Host "No zip files matching pattern '\$zipPattern' found in '\$PWD'." - # Exit PowerShell script with a non-zero code to fail the step + dir("C:/Jenkins/workspace/AzaionSuite/suite") { + def output = powershell(returnStdout: true, script: ''' + $pattern = "AzaionSuite*-*-*.zip" + $zipFiles = Get-ChildItem -Filter $pattern | Sort-Object Name -Descending + if ($zipFiles.Count -eq 0) { + Write-Error "No ZIP files matching pattern '$pattern' were found." exit 1 } + $latestZip = $zipFiles[0].Name + Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$latestZip" + ''').trim() - Write-Host "Found \$(\$zipFiles.Count) zip file(s)." - - # --- Sorting of ZIP Files by Filename Timestamp (in PowerShell) --- - Write-Host "Sorting ZIP files by filename timestamp (YYYYMMDD-HHMMSS)..." - - # Regex to extract the timestamp from the filename - \$timestampRegex = '.*-(\\d{8}-\\d{6})\\.zip' - - # Sort the files by extracting the timestamp and converting to DateTime for accurate sorting - # Sort-Object -Descending ensures newest are first - \$sortedZipFiles = \$zipFiles | Sort-Object -Descending { - \$name = \$_.Name - \$match = [regex]::Match(\$name, \$timestampRegex) - if (\$match.Success -and \$match.Groups.Count -gt 1) { - \$timestampStr = \$match.Groups[1].Value - # Attempt to parse the timestamp string into a DateTime object - try { - [DateTime]::ParseExact(\$timestampStr, "yyyyMMdd-HHmmss", \$null) - } catch { - Write-Host "Warning: Could not parse timestamp from filename '\$name': \$(\$_.Exception.Message)" - # Handle parsing errors - treat as the oldest possible date (e.g., 1/1/0001) - # This ensures unparseable dates are placed at the end (oldest) - [DateTime]::MinValue - } - } else { - Write-Host "Warning: Filename '\$name' does not match timestamp regex." - # Handle non-matching filenames - treat as the oldest possible date - [DateTime]::MinValue - # Exit the script here if you want to fail the stage on unparseable filenames - # exit 1 - } - } - # --- End Sorting --- - - # Get the name of the latest zip file (the first one after sorting descending) - \$latestZipFile = \$sortedZipFiles | Select-Object -First 1 - - Write-Host "Identified latest zip file: '\$latestZipFile.Name'" - - # Output the latest zip filename and set it as an environment variable for Jenkins - # This uses the Jenkins 'set context' feature - Write-Host "::SET-ENV::LATEST_ZIP_FILENAME=\$(\$latestZipFile.Name)" - - exit 0 # Exit PowerShell script successfully - - """ , returnStdout: true - ) // End powershell call - - // Parse the PowerShell output to find the latest zip filename and set the Groovy environment variable - def matcher = powershellOutput =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/ - def latestZipFilename = null - if (matcher.find()) { - latestZipFilename = matcher.group(1).trim() - env.LATEST_ZIP_FILENAME = latestZipFilename // Set the environment variable - echo "Groovy set LATEST_ZIP_FILENAME to: ${env.LATEST_ZIP_FILENAME}" - } else { - error "Could not find the latest zip filename in the PowerShell output using marker." + def match = output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/ + if (!match) { + error("Could not find the latest zip filename in the PowerShell output using marker.") } + def zipFileName = match[0][1] + echo "Latest zip file selected: ${zipFileName}" - // Ensure the latest zip filename environment variable is set before attempting upload - if (env.LATEST_ZIP_FILENAME && env.LATEST_ZIP_FILENAME != '') { - echo "Identified latest zip file for upload: ${env.LATEST_ZIP_FILENAME}" - - try { - // --- Upload the latest ZIP archive directly from the current directory --- - echo "Starting upload of '${env.LATEST_ZIP_FILENAME}' directly from '${pwd()}' to ${GDRIVE_REMOTE}..." - // Use standard Windows PowerShell to execute rclone (now in PATH) - // Ensure config path and remote path are quoted, and use backslashes for Windows paths - // The source path for rclone is the filename relative to the current directory (TARGET_ARTIFACTS_DIR) - powershell """ - rclone --config "${env:WORKSPACE}\\\\${env:RCLONE_CONFIG}" copy \"${env:LATEST_ZIP_FILENAME}\" \"${env:GDRIVE_REMOTE}\" - """ - echo "Finished uploading ${env.LATEST_ZIP_FILENAME}." - - } catch (e) { - echo "ERROR uploading build: ${e}" - error("Failed to upload latest zip file: ${e.message}") // Fail the stage on error - } - // No cleanup needed in finally block here as the file is not copied to a temp location first - } else { - error "LATEST_ZIP_FILENAME environment variable was not set or was empty. Cannot upload." - } - } // End dir block - } // end script - } // end steps - } // end stage + // Pass the variable into the environment for the next steps if needed + env.LATEST_ZIP_FILENAME = zipFileName + } + } + } + } stage('Retention on Google Drive') { - steps { - script { // Wrap script logic in a script block - echo "Starting Google Drive retention process (using PowerShell)..." - // Ensure rclone is installed and in PATH on the Windows agent, - // and the Jenkins agent user has read access to '${env:WORKSPACE}\\\\${env.RCLONE_CONFIG}'. - - // PowerShell script block for retention logic - powershell """ - \$rcloneRemote = "${env:GDRIVE_REMOTE}" - \$rcloneConfig = "${env:WORKSPACE}\\\\${env:RCLONE_CONFIG}" - \$filesToKeep = ${env.FILES_TO_KEEP} - - # Get list of files from Google Drive as JSON - \$rcloneListOutput = rclone --config "\$rcloneConfig" lsjson "\$rcloneRemote" | Out-String - - # Parse JSON output - # ConvertFrom-Json will throw an error if the input is not valid JSON, - # which will cause the PowerShell step and the stage to fail. - \$allFilesJson = \$rcloneListOutput | ConvertFrom-Json - - # Filter for zip files - \$zipFiles = \$allFilesJson | Where-Object { \$_.Name -ne \$null -and \$_.Name.EndsWith(".zip") } - - Write-Host "Found \$(\$zipFiles.Count) total ZIP files on Google Drive." - - # --- Sorting of ZIP Files by Filename Timestamp (in PowerShell) --- - Write-Host "Sorting ZIP files on Google Drive by filename timestamp (YYYYMMDD-HHMMSS)..." - - # Regex to extract the timestamp from the filename - \$timestampRegex = '.*-(\\d{8}-\\d{6})\\.zip' - - # Sort the files by extracting the timestamp and converting to DateTime for accurate sorting - # Sort-Object -Descending ensures newest are first - \$sortedZipFiles = \$zipFiles | Sort-Object -Descending { - \$name = \$_.Name - \$match = [regex]::Match(\$name, \$timestampRegex) - if (\$match.Success -and \$match.Groups.Count -gt 1) { - \$timestampStr = \$match.Groups[1].Value - # Attempt to parse the timestamp string into a DateTime object - try { - [DateTime]::ParseExact(\$timestampStr, "yyyyMMdd-HHmmss", \$null) - } catch { - Write-Host "Warning: Could not parse timestamp from filename '\$name': \$(\$_.Exception.Message)" - # Handle parsing errors - treat as the oldest possible date (e.g., 1/1/0001) - # This ensures unparseable dates are placed at the end (oldest) - [DateTime]::MinValue - } - } else { - Write-Host "Warning: Filename '\$name' does not match timestamp regex." - # Handle non-matching filenames - treat as the oldest possible date - [DateTime]::MinValue - # Exit the script here if you want to fail the stage on unparseable filenames - # exit 1 - } - } - # --- End Sorting --- - - # DEBUG: Print the sorted list by filename timestamp with each file on a new line - Write-Host "DEBUG: ZIP files on Google Drive sorted by filename timestamp (newest first):" - \$sortedZipFiles | ForEach-Object { Write-Host \$_.Name } - - - # Keep the latest N files, identify the rest for deletion - if (\$sortedZipFiles.Count -gt \$filesToKeep) { - # Select the files to delete (from index FILES_TO_KEEP to the end) - \$filesToDelete = \$sortedZipFiles | Select-Object -Skip \$filesToKeep - - Write-Host "Applying retention: Keeping \$filesToKeep newest files, deleting \$(\$filesToDelete.Count) older files." - # DEBUG: Print the list of files identified for deletion - Write-Host "DEBUG: Files identified for deletion:" - \$filesToDelete | ForEach-Object { Write-Host \$_.Name } - - - # Loop through files to delete and execute rclone delete - foreach (\$oldZipInfo in \$filesToDelete) { - \$oldZipName = \$oldZipInfo.Name - Write-Host "Deleting old ZIP from Google Drive: \$oldZipName" - # Ensure filenames are quoted for safety, especially if they contain spaces - # Use errorHanding: 'ignore' if you want to continue even if a delete fails - rclone --config "\$rcloneConfig" delete "\$rcloneRemote/\$oldZipName" --drive-use-trash=false - } - } else { - Write-Host "Retention check: Found \$(\$sortedZipFiles.Count) ZIP files, which is not more than \$filesToKeep. No files deleted." - } - """ // End PowerShell script block - } // end script - } // end steps - } // end stage - - } // End of main 'stages' block + steps { + echo "Uploading ${env.LATEST_ZIP_FILENAME} to Google Drive..." + // Example command (replace with actual upload command) + powershell """ + \$filePath = "C:/Jenkins/workspace/AzaionSuite/suite/${env.LATEST_ZIP_FILENAME}" + Write-Output "Would upload: \$filePath" + # Add your actual GDrive upload logic here + """ + } + } + } post { always { - script { // Wrap steps in a script block - echo "Executing post-build cleanup..." - // Use standard Windows PowerShell Remove-Item for cleanup - // Quote TMP_UPLOAD_DIR path for safety and use backslashes for Windows paths - powershell """ - echo 'Cleaning up temporary upload directory: ${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}' - Remove-Item -Recurse -Force "${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}" -ErrorAction SilentlyContinue - """ - // Removed cleanup for copied artifacts directory as files are cleaned up individually now. - } // end script + echo 'Executing post-build cleanup...' + powershell ''' + $uploadDir = "temp_upload" + if (Test-Path $uploadDir) { + Remove-Item -Recurse -Force $uploadDir + } + ''' } - success { - script { // Wrap steps in a script block - echo "Pipeline finished successfully." - // Add any success-specific notifications or actions here - } // end script - } - failure { - script { // Wrap steps in a script block - echo "Pipeline failed. Check logs for details." - // Add any failure-specific notifications or actions here - } // end script - } - } // End of 'post' block -} // End of 'pipeline' block + failure { + echo 'Pipeline failed. Check logs for details.' + } + } +}