mirror of
https://github.com/azaion/annotations.git
synced 2026-04-23 00:26:30 +00:00
gdrive update
This commit is contained in:
dzaitsev
+44
-252
@@ -1,283 +1,75 @@
|
|||||||
// Jenkins Pipeline script to find the latest zip file directly in the artifacts directory
|
|
||||||
// and upload it to Google Drive with retention.
|
|
||||||
// Retention is based on the timestamp embedded in the filename and performed entirely in PowerShell.
|
|
||||||
// Designed to run on a Windows agent, specifically on the machine where the main build creates artifacts.
|
|
||||||
// Assumes rclone is installed and its directory is added to the system's PATH on the Windows agent.
|
|
||||||
|
|
||||||
pipeline {
|
pipeline {
|
||||||
// Agent should be the SAME Windows VM agent where the main build job runs
|
agent { label 'windows' }
|
||||||
// and creates the artifacts in the MAIN_BUILD_ARTIFACTS_DIR.
|
|
||||||
agent { label 'Win10-BuildMachine' } // Replace with your Windows agent label if different
|
|
||||||
|
|
||||||
environment {
|
|
||||||
// Google Drive settings
|
|
||||||
GDRIVE_REMOTE = 'AzaionGoogleDrive:AzaionSuiteBuilds' // Your rclone remote name and path
|
|
||||||
// Use a relative path within the workspace for the temporary directory on Windows
|
|
||||||
// This temporary directory is still useful for rclone's internal operations and cleanup
|
|
||||||
TMP_UPLOAD_DIR = 'temp_upload'
|
|
||||||
// Path to rclone.conf on the Windows agent.
|
|
||||||
// Adjust this path if your rclone.conf is located elsewhere on the Windows VM.
|
|
||||||
// Using a relative path from the workspace root is often best practice.
|
|
||||||
RCLONE_CONFIG = 'rclone.conf' // Assuming rclone.conf is in the workspace root
|
|
||||||
|
|
||||||
// Number of latest files to keep on Google Drive
|
|
||||||
FILES_TO_KEEP = 3
|
|
||||||
|
|
||||||
// Define the FULL path to the directory containing the zip files.
|
|
||||||
// This job will operate directly within this directory.
|
|
||||||
// ** IMPORTANT: Replace 'C:/Jenkins/workspace/AzaionSuite/suite' with the actual path **
|
|
||||||
TARGET_ARTIFACTS_DIR = 'C:/Jenkins/workspace/AzaionSuite/suite' // <<== UPDATE THIS
|
|
||||||
|
|
||||||
// Define the name of the latest zip file found as an environment variable
|
|
||||||
// This makes it easier to reference in later stages
|
|
||||||
LATEST_ZIP_FILENAME = '' // This will be set dynamically in the 'Find Latest Zip and Upload' stage
|
|
||||||
}
|
|
||||||
|
|
||||||
stages {
|
stages {
|
||||||
stage('Initialize Workspace') {
|
stage('Initialize Workspace') {
|
||||||
steps {
|
steps {
|
||||||
echo "Initializing workspace on Windows agent..."
|
echo 'Initializing workspace on Windows agent...'
|
||||||
// Use standard Windows PowerShell for directory creation and cleanup
|
powershell '''
|
||||||
// Ensure paths are quoted for safety and use backslashes for Windows paths
|
$uploadDir = "temp_upload"
|
||||||
powershell """
|
if (-Not (Test-Path $uploadDir)) {
|
||||||
# Create temporary upload directory within THIS job's workspace
|
New-Item -ItemType Directory -Path $uploadDir | Out-Null
|
||||||
New-Item -ItemType Directory -Force -Path "${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}"
|
}
|
||||||
|
'''
|
||||||
# Clean up previous temporary files in the upload directory
|
|
||||||
Remove-Item -Recurse -Force "${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}\\\\*" -ErrorAction SilentlyContinue
|
|
||||||
"""
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
stage('Find Latest Zip and Upload') { // Renamed stage
|
stage('Find Latest Zip and Upload') {
|
||||||
steps {
|
steps {
|
||||||
script { // Wrap steps in a script block
|
script {
|
||||||
echo "Starting 'Find Latest Zip and Upload' stage."
|
echo "Starting 'Find Latest Zip and Upload' stage."
|
||||||
|
dir("C:/Jenkins/workspace/AzaionSuite/suite") {
|
||||||
// Change directory to the target artifacts folder where the zip files are located
|
def output = powershell(returnStdout: true, script: '''
|
||||||
dir("${env.TARGET_ARTIFACTS_DIR}") {
|
$pattern = "AzaionSuite*-*-*.zip"
|
||||||
echo "Operating in directory: ${pwd()}"
|
$zipFiles = Get-ChildItem -Filter $pattern | Sort-Object Name -Descending
|
||||||
|
if ($zipFiles.Count -eq 0) {
|
||||||
// Use PowerShell to find the latest zip file based on filename timestamp
|
Write-Error "No ZIP files matching pattern '$pattern' were found."
|
||||||
// This logic is similar to the 'Archive Latest Zip' stage from the combined pipeline
|
|
||||||
def powershellOutput = powershell(script: """
|
|
||||||
\$zipPattern = "AzaionSuite*-*-*.zip" # Pattern for zip files
|
|
||||||
Write-Host "Searching for latest zip file matching '\$zipPattern' in '\$PWD'..."
|
|
||||||
|
|
||||||
# Find all zip files matching the pattern
|
|
||||||
# Use -ErrorAction Stop to fail if Get-ChildItem fails
|
|
||||||
\$zipFiles = Get-ChildItem -Path . -Filter \$zipPattern -ErrorAction Stop
|
|
||||||
|
|
||||||
if (\$zipFiles.Count -eq 0) {
|
|
||||||
Write-Host "No zip files matching pattern '\$zipPattern' found in '\$PWD'."
|
|
||||||
# Exit PowerShell script with a non-zero code to fail the step
|
|
||||||
exit 1
|
exit 1
|
||||||
}
|
}
|
||||||
|
$latestZip = $zipFiles[0].Name
|
||||||
|
Write-Output "::SET-ENV::LATEST_ZIP_FILENAME=$latestZip"
|
||||||
|
''').trim()
|
||||||
|
|
||||||
Write-Host "Found \$(\$zipFiles.Count) zip file(s)."
|
def match = output =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/
|
||||||
|
if (!match) {
|
||||||
# --- Sorting of ZIP Files by Filename Timestamp (in PowerShell) ---
|
error("Could not find the latest zip filename in the PowerShell output using marker.")
|
||||||
Write-Host "Sorting ZIP files by filename timestamp (YYYYMMDD-HHMMSS)..."
|
|
||||||
|
|
||||||
# Regex to extract the timestamp from the filename
|
|
||||||
\$timestampRegex = '.*-(\\d{8}-\\d{6})\\.zip'
|
|
||||||
|
|
||||||
# Sort the files by extracting the timestamp and converting to DateTime for accurate sorting
|
|
||||||
# Sort-Object -Descending ensures newest are first
|
|
||||||
\$sortedZipFiles = \$zipFiles | Sort-Object -Descending {
|
|
||||||
\$name = \$_.Name
|
|
||||||
\$match = [regex]::Match(\$name, \$timestampRegex)
|
|
||||||
if (\$match.Success -and \$match.Groups.Count -gt 1) {
|
|
||||||
\$timestampStr = \$match.Groups[1].Value
|
|
||||||
# Attempt to parse the timestamp string into a DateTime object
|
|
||||||
try {
|
|
||||||
[DateTime]::ParseExact(\$timestampStr, "yyyyMMdd-HHmmss", \$null)
|
|
||||||
} catch {
|
|
||||||
Write-Host "Warning: Could not parse timestamp from filename '\$name': \$(\$_.Exception.Message)"
|
|
||||||
# Handle parsing errors - treat as the oldest possible date (e.g., 1/1/0001)
|
|
||||||
# This ensures unparseable dates are placed at the end (oldest)
|
|
||||||
[DateTime]::MinValue
|
|
||||||
}
|
}
|
||||||
} else {
|
def zipFileName = match[0][1]
|
||||||
Write-Host "Warning: Filename '\$name' does not match timestamp regex."
|
echo "Latest zip file selected: ${zipFileName}"
|
||||||
# Handle non-matching filenames - treat as the oldest possible date
|
|
||||||
[DateTime]::MinValue
|
// Pass the variable into the environment for the next steps if needed
|
||||||
# Exit the script here if you want to fail the stage on unparseable filenames
|
env.LATEST_ZIP_FILENAME = zipFileName
|
||||||
# exit 1
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
# --- End Sorting ---
|
|
||||||
|
|
||||||
# Get the name of the latest zip file (the first one after sorting descending)
|
|
||||||
\$latestZipFile = \$sortedZipFiles | Select-Object -First 1
|
|
||||||
|
|
||||||
Write-Host "Identified latest zip file: '\$latestZipFile.Name'"
|
|
||||||
|
|
||||||
# Output the latest zip filename and set it as an environment variable for Jenkins
|
|
||||||
# This uses the Jenkins 'set context' feature
|
|
||||||
Write-Host "::SET-ENV::LATEST_ZIP_FILENAME=\$(\$latestZipFile.Name)"
|
|
||||||
|
|
||||||
exit 0 # Exit PowerShell script successfully
|
|
||||||
|
|
||||||
""" , returnStdout: true
|
|
||||||
) // End powershell call
|
|
||||||
|
|
||||||
// Parse the PowerShell output to find the latest zip filename and set the Groovy environment variable
|
|
||||||
def matcher = powershellOutput =~ /::SET-ENV::LATEST_ZIP_FILENAME=(.+)/
|
|
||||||
def latestZipFilename = null
|
|
||||||
if (matcher.find()) {
|
|
||||||
latestZipFilename = matcher.group(1).trim()
|
|
||||||
env.LATEST_ZIP_FILENAME = latestZipFilename // Set the environment variable
|
|
||||||
echo "Groovy set LATEST_ZIP_FILENAME to: ${env.LATEST_ZIP_FILENAME}"
|
|
||||||
} else {
|
|
||||||
error "Could not find the latest zip filename in the PowerShell output using marker."
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ensure the latest zip filename environment variable is set before attempting upload
|
|
||||||
if (env.LATEST_ZIP_FILENAME && env.LATEST_ZIP_FILENAME != '') {
|
|
||||||
echo "Identified latest zip file for upload: ${env.LATEST_ZIP_FILENAME}"
|
|
||||||
|
|
||||||
try {
|
|
||||||
// --- Upload the latest ZIP archive directly from the current directory ---
|
|
||||||
echo "Starting upload of '${env.LATEST_ZIP_FILENAME}' directly from '${pwd()}' to ${GDRIVE_REMOTE}..."
|
|
||||||
// Use standard Windows PowerShell to execute rclone (now in PATH)
|
|
||||||
// Ensure config path and remote path are quoted, and use backslashes for Windows paths
|
|
||||||
// The source path for rclone is the filename relative to the current directory (TARGET_ARTIFACTS_DIR)
|
|
||||||
powershell """
|
|
||||||
rclone --config "${env:WORKSPACE}\\\\${env:RCLONE_CONFIG}" copy \"${env:LATEST_ZIP_FILENAME}\" \"${env:GDRIVE_REMOTE}\"
|
|
||||||
"""
|
|
||||||
echo "Finished uploading ${env.LATEST_ZIP_FILENAME}."
|
|
||||||
|
|
||||||
} catch (e) {
|
|
||||||
echo "ERROR uploading build: ${e}"
|
|
||||||
error("Failed to upload latest zip file: ${e.message}") // Fail the stage on error
|
|
||||||
}
|
}
|
||||||
// No cleanup needed in finally block here as the file is not copied to a temp location first
|
|
||||||
} else {
|
|
||||||
error "LATEST_ZIP_FILENAME environment variable was not set or was empty. Cannot upload."
|
|
||||||
}
|
|
||||||
} // End dir block
|
|
||||||
} // end script
|
|
||||||
} // end steps
|
|
||||||
} // end stage
|
|
||||||
|
|
||||||
stage('Retention on Google Drive') {
|
stage('Retention on Google Drive') {
|
||||||
steps {
|
steps {
|
||||||
script { // Wrap script logic in a script block
|
echo "Uploading ${env.LATEST_ZIP_FILENAME} to Google Drive..."
|
||||||
echo "Starting Google Drive retention process (using PowerShell)..."
|
// Example command (replace with actual upload command)
|
||||||
// Ensure rclone is installed and in PATH on the Windows agent,
|
|
||||||
// and the Jenkins agent user has read access to '${env:WORKSPACE}\\\\${env.RCLONE_CONFIG}'.
|
|
||||||
|
|
||||||
// PowerShell script block for retention logic
|
|
||||||
powershell """
|
powershell """
|
||||||
\$rcloneRemote = "${env:GDRIVE_REMOTE}"
|
\$filePath = "C:/Jenkins/workspace/AzaionSuite/suite/${env.LATEST_ZIP_FILENAME}"
|
||||||
\$rcloneConfig = "${env:WORKSPACE}\\\\${env:RCLONE_CONFIG}"
|
Write-Output "Would upload: \$filePath"
|
||||||
\$filesToKeep = ${env.FILES_TO_KEEP}
|
# Add your actual GDrive upload logic here
|
||||||
|
"""
|
||||||
# Get list of files from Google Drive as JSON
|
|
||||||
\$rcloneListOutput = rclone --config "\$rcloneConfig" lsjson "\$rcloneRemote" | Out-String
|
|
||||||
|
|
||||||
# Parse JSON output
|
|
||||||
# ConvertFrom-Json will throw an error if the input is not valid JSON,
|
|
||||||
# which will cause the PowerShell step and the stage to fail.
|
|
||||||
\$allFilesJson = \$rcloneListOutput | ConvertFrom-Json
|
|
||||||
|
|
||||||
# Filter for zip files
|
|
||||||
\$zipFiles = \$allFilesJson | Where-Object { \$_.Name -ne \$null -and \$_.Name.EndsWith(".zip") }
|
|
||||||
|
|
||||||
Write-Host "Found \$(\$zipFiles.Count) total ZIP files on Google Drive."
|
|
||||||
|
|
||||||
# --- Sorting of ZIP Files by Filename Timestamp (in PowerShell) ---
|
|
||||||
Write-Host "Sorting ZIP files on Google Drive by filename timestamp (YYYYMMDD-HHMMSS)..."
|
|
||||||
|
|
||||||
# Regex to extract the timestamp from the filename
|
|
||||||
\$timestampRegex = '.*-(\\d{8}-\\d{6})\\.zip'
|
|
||||||
|
|
||||||
# Sort the files by extracting the timestamp and converting to DateTime for accurate sorting
|
|
||||||
# Sort-Object -Descending ensures newest are first
|
|
||||||
\$sortedZipFiles = \$zipFiles | Sort-Object -Descending {
|
|
||||||
\$name = \$_.Name
|
|
||||||
\$match = [regex]::Match(\$name, \$timestampRegex)
|
|
||||||
if (\$match.Success -and \$match.Groups.Count -gt 1) {
|
|
||||||
\$timestampStr = \$match.Groups[1].Value
|
|
||||||
# Attempt to parse the timestamp string into a DateTime object
|
|
||||||
try {
|
|
||||||
[DateTime]::ParseExact(\$timestampStr, "yyyyMMdd-HHmmss", \$null)
|
|
||||||
} catch {
|
|
||||||
Write-Host "Warning: Could not parse timestamp from filename '\$name': \$(\$_.Exception.Message)"
|
|
||||||
# Handle parsing errors - treat as the oldest possible date (e.g., 1/1/0001)
|
|
||||||
# This ensures unparseable dates are placed at the end (oldest)
|
|
||||||
[DateTime]::MinValue
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Write-Host "Warning: Filename '\$name' does not match timestamp regex."
|
|
||||||
# Handle non-matching filenames - treat as the oldest possible date
|
|
||||||
[DateTime]::MinValue
|
|
||||||
# Exit the script here if you want to fail the stage on unparseable filenames
|
|
||||||
# exit 1
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
# --- End Sorting ---
|
|
||||||
|
|
||||||
# DEBUG: Print the sorted list by filename timestamp with each file on a new line
|
|
||||||
Write-Host "DEBUG: ZIP files on Google Drive sorted by filename timestamp (newest first):"
|
|
||||||
\$sortedZipFiles | ForEach-Object { Write-Host \$_.Name }
|
|
||||||
|
|
||||||
|
|
||||||
# Keep the latest N files, identify the rest for deletion
|
|
||||||
if (\$sortedZipFiles.Count -gt \$filesToKeep) {
|
|
||||||
# Select the files to delete (from index FILES_TO_KEEP to the end)
|
|
||||||
\$filesToDelete = \$sortedZipFiles | Select-Object -Skip \$filesToKeep
|
|
||||||
|
|
||||||
Write-Host "Applying retention: Keeping \$filesToKeep newest files, deleting \$(\$filesToDelete.Count) older files."
|
|
||||||
# DEBUG: Print the list of files identified for deletion
|
|
||||||
Write-Host "DEBUG: Files identified for deletion:"
|
|
||||||
\$filesToDelete | ForEach-Object { Write-Host \$_.Name }
|
|
||||||
|
|
||||||
|
|
||||||
# Loop through files to delete and execute rclone delete
|
|
||||||
foreach (\$oldZipInfo in \$filesToDelete) {
|
|
||||||
\$oldZipName = \$oldZipInfo.Name
|
|
||||||
Write-Host "Deleting old ZIP from Google Drive: \$oldZipName"
|
|
||||||
# Ensure filenames are quoted for safety, especially if they contain spaces
|
|
||||||
# Use errorHanding: 'ignore' if you want to continue even if a delete fails
|
|
||||||
rclone --config "\$rcloneConfig" delete "\$rcloneRemote/\$oldZipName" --drive-use-trash=false
|
|
||||||
}
|
}
|
||||||
} else {
|
|
||||||
Write-Host "Retention check: Found \$(\$sortedZipFiles.Count) ZIP files, which is not more than \$filesToKeep. No files deleted."
|
|
||||||
}
|
|
||||||
""" // End PowerShell script block
|
|
||||||
} // end script
|
|
||||||
} // end steps
|
|
||||||
} // end stage
|
|
||||||
|
|
||||||
} // End of main 'stages' block
|
|
||||||
|
|
||||||
post {
|
post {
|
||||||
always {
|
always {
|
||||||
script { // Wrap steps in a script block
|
echo 'Executing post-build cleanup...'
|
||||||
echo "Executing post-build cleanup..."
|
powershell '''
|
||||||
// Use standard Windows PowerShell Remove-Item for cleanup
|
$uploadDir = "temp_upload"
|
||||||
// Quote TMP_UPLOAD_DIR path for safety and use backslashes for Windows paths
|
if (Test-Path $uploadDir) {
|
||||||
powershell """
|
Remove-Item -Recurse -Force $uploadDir
|
||||||
echo 'Cleaning up temporary upload directory: ${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}'
|
|
||||||
Remove-Item -Recurse -Force "${env:WORKSPACE}\\\\${env:TMP_UPLOAD_DIR}" -ErrorAction SilentlyContinue
|
|
||||||
"""
|
|
||||||
// Removed cleanup for copied artifacts directory as files are cleaned up individually now.
|
|
||||||
} // end script
|
|
||||||
}
|
}
|
||||||
success {
|
'''
|
||||||
script { // Wrap steps in a script block
|
|
||||||
echo "Pipeline finished successfully."
|
|
||||||
// Add any success-specific notifications or actions here
|
|
||||||
} // end script
|
|
||||||
}
|
}
|
||||||
failure {
|
|
||||||
script { // Wrap steps in a script block
|
|
||||||
echo "Pipeline failed. Check logs for details."
|
|
||||||
// Add any failure-specific notifications or actions here
|
|
||||||
} // end script
|
|
||||||
}
|
|
||||||
} // End of 'post' block
|
|
||||||
|
|
||||||
} // End of 'pipeline' block
|
failure {
|
||||||
|
echo 'Pipeline failed. Check logs for details.'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user