Mageia Bugzilla – Attachment 11685 Details for
Bug 26745
ISO_testing_rsync_tools#Onesync refers to script with no place to download it from.
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
Log In
[x]
|
New Account
|
Forgot Password
onsync script for qa iso testers
onesync (text/plain), 10.49 KB, created by
Dave Hodgins
on 2020-06-08 20:46:24 CEST
(
hide
)
Description:
onsync script for qa iso testers
Filename:
MIME Type:
Creator:
Dave Hodgins
Created:
2020-06-08 20:46:24 CEST
Size:
10.49 KB
patch
obsolete
>#!/bin/bash ># An attempt at a generic ISO rsync update script which copes with file re-naming ># onesync -h | -i | DIR ># -h Ouput command help ># -i Just list remote ISO directory ># DIR The local ISO directory to synchronise ># . for the current directory ># If it does not exist, offer to create it > ># DEFINE THIS AS NECESSARY & WHEN IT CHANGES >RSYNC_PASSWORD="munged" ### <<< UPDATE THIS <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<!!!!!!!!!!!!!!!! >export RSYNC_PASSWORD > ># THE FOLLOWING ITEMS *COULD* CHANGE but are normally constant ># rsync user >USER="isoqa" ># ISOs host/source directory >SOURCE="bcd.mageia.org/isos" > >USAGE="Usage: onesync -h | -i | DIR" > ># Check that we have just 1 parameter >if [ $# -ne 1 ] >then echo $USAGE ; exit ; fi > ># http://alvinalexander.com/linux-unix/shell-script-command-line-arguments-linux-unix ># Look for -h help or -i info option >while getopts :hi OPT >do > ># >>>>>>>> > >case $OPT in ># -h for Help, output that and exit > h) > echo $USAGE > echo -e "-h \t Ouput this help" > echo -e "-i \t List a remote directory" > echo -e "DIR \t The local ISO directory to synchronise" > echo -e "\t '.' for the current directory" > echo -e "\t If the directory does not exist, it will be optionally created" > echo > echo -e "\t Examples:" > echo onesync -h > echo " shows this help text" > echo > echo onesync -i > echo " enables listing a chosen remote ISO" > echo > echo onesync . > echo " synchronises the [ISO] directory it is issued from" > echo > echo "onesync Mageia/iso/Classic64 [The directory exists]" > echo " synchronises the existing [ISO] directory Mageia/iso/Classic64/" > echo > echo "onesync Mageia/iso/Classic64 [The directory does not yet exist]" > echo " optionally creates & primes the [ISO] directory Mageia/iso/Classic64/" > echo > exit > ;; > ># -i for just remote directory listing simply set INFO flag > i) > INFO=1 > ;; > > *) > echo Invalid parameter > exit >esac > ># <<<<<<<< > >done > >echo > ># If called just for info, skip most parts & exit early > >if [ $INFO ] >then > echo "This script lists the contents of a chosen ISO on the server" >else > ># Real synchronisation ># >>>>>>>> This part skipped for info-only; should all be indented > >echo "This script primes or updates a single ISO directory" >echo > ># Save where we are called from, to return to at the end if necessary >FROM=`pwd` ># Get the local directory parameter. '.' means *this* directory >if [ $1 != '.' ] >then > DIR=$1 ># Check the directory exists; if not, offer to create it > if [ ! -d $1 ] > then > echo The specified directory $1 does not exist. > echo -n "<Enter> to create it, anything else to quit: " > read REPLY ># $REPLY needs to be quoted to recognise a null return ># Continue only if asked to do so, else quit > if [ "$REPLY" != "" ] ; then exit ; fi ># Create the local ISO directory > mkdir $1 > fi ># Go to the given directory (if it was not '.') > cd $DIR >fi > ># Temporary file to hold list of files in the ISO directory >ISOFILES=/tmp/ISOfiles`date +%H%M%S` > ># Confirm that we are in the correct local directory (which can have any name) >HERE=`pwd` >echo "You should now be in the local ISO directory you want to update:" >echo -e "$HERE" > ># $5=size without commas, $6=coerced yyyy/mm/dd, $7=coerced hh:mm, $8=filename >ls -l --time-style='+%Y/%m/%d %H:%M' | awk 'NR>1 {printf "%10s %s %s %s\n", $5, $6, $7, $8}' > >echo -n "If this is correct, <Enter> to continue; anything else to quit: " >read REPLY ># $REPLY needs to be quoted to recognise a null return ># Continue only if asked to do so, else quit >if [ "$REPLY" != "" ] ; then exit ; fi > ># <<<<<<<< End of part skipped for info-only; should have been indented > >fi > ># Start of part shared by info > >echo ># First test the rsync password. Grep should return 0 for 'found' >rsync --list-only rsync://${USER}@$SOURCE/ 2>&1 | grep 'auth failed' >if [ $? -eq 0 ] >then > echo The current script rsync password \'$RSYNC_PASSWORD\' seems to be wrong. > echo Please update it in the onesync script > exit >else > echo The current script rsync password \'$RSYNC_PASSWORD\' seems OK >fi > ># Read current release umbrella directories >echo >echo "These are the current releases; choose the one you want:" >select PHASE in `rsync --list-only rsync://${USER}@$SOURCE/ | awk 'NR>1 {print $5}'` >do ># A null reply automatically repeats the prompt ># Otherwise, check for out-of-range reply, PHASE = NUL; abort if so. > if [ "$PHASE" == "" ] ; then exit ; fi > echo -e "\t Chosen release is: $PHASE" > break >done ># In case they responded ^D (EOF) which exits the do loop above to here >if [ "$PHASE" == "" ] ; then exit ; fi > ># Read current ISO directories for chosen release >echo "These are its current ISOs; choose CAREFULLY:" >select ISODIR in `rsync --list-only rsync://${USER}@$SOURCE/$PHASE/ | awk 'NR>1 {print $5}'` >do ># A null reply automatically repeats the prompt ># Otherwise, check for out-of-range reply, ISODIR = NUL; abort if so. > if [ "$ISODIR" == "" ] ; then exit ; fi > echo -e "\t Chosen ISO is: $ISODIR" > echo Its current content: > break >done ># In case they responded ^D (EOF) which exits the do loop above to here >if [ "$ISODIR" == "" ] ; then exit ; fi > ># Display for info the remote ISO directory contents (skip directory . line) ># The tr removes commas (from and only in file size) ># $2=size without commas, $3=yyyy/mm/dd, $4=hh:mm:ss cut seconds, $5=filename > >rsync --list-only rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ \ > | tr -d ',' \ > | awk 'NR>1 {printf "%10s %s %.5s %s\n", $2, $3, $4, $5}' > ># End of part shared by info; just exit > >if [ $INFO ] ; then echo; exit; fi > >echo -n "Does it look OK? <Enter> to confirm, anything else to quit: " >read REPLY ># $REPLY needs to be quoted to recognise a null return ># Continue only if asked to do so, else quit >if [ "$REPLY" != "" ] ; then exit ; fi > ># Record filenames in the chosen ISO directory to compare with their local names >rsync --list-only rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ \ > | awk 'NR>1 {print $5}' > $ISOFILES > >echo >echo "Each filename in the remote directory is checked against its" >echo "equivalent (based on its extension) in this local one." ># THE FILENAME CONSISTENCY ALGORITHM REQUIRES ONLY 1 FILE OF EACH EXTENSION >echo If a discrepancy is found, you will be asked to confirm renaming the local copy. >echo This is to cope with release filename changes. >echo No local equivalent implies a virgin download. >echo > ># Overall record of any local filename changes ># FLAG Some re-naming to do ># ALL Rename all local files ># ERR >1 file of same extension > ># Work through the list of remote filenames >for F in `cat $ISOFILES` >do ># The remote filename > REMOTE=$F ># If the remote filename already exists locally, skip it > if [ -f $REMOTE ] ; then echo $REMOTE will be updated ; continue ; fi > ># We did not find an exact match; find the current equivalent local file based on the extension ># BASE=${F%.*} ># Its extension > EXTN=${F##*.} ># See if just 1 local file of same extension exists to rename > COUNT=`ls *.$EXTN 2>/dev/null | wc -l` ># If no equivalent local file, none to rename, it will be downloaded > if [ $COUNT -eq 0 ] ; then echo $REMOTE will be downloaded ; continue ; fi ># The equivalent local filename(s) > LOCAL=`ls *.$EXTN` ># If there is >1 filename of this extension, complain & skip them > if [ $COUNT -gt 1 ] > then > ERR=1 > echo There is more than one local file of type .$EXTN > echo -e "The following local files are NOT renamed: \n $LOCAL" > continue > fi > ># We have a filename which can & should be brought up-to-date ># Rename the local file to be the same as its remote (extension) equivalent ># Only query them if not already asked to do all files. > if [ $ALL ] ># They want all renamed without confirmation > then > echo -e "Renaming for update local file \n $LOCAL to \n $REMOTE" > else ># Ask their opinion > FLAG=1 > echo > echo -e "About to rename for update local file \n $LOCAL \t to \n $REMOTE" > echo "<Enter> alone to rename ALL files without asking again," > echo " r|R to rename just THIS file," > echo -n " anything else to leave this filename as it is: " > read REPLY > ># Rename only if asked to do so, otherwise skip this file > # $REPLY needs to be quoted to recognise a null return > if [ "$REPLY" == "" ] ># Re-naming without further confirmation > then ALL=1 > elif [ "$REPLY" != "r" -a "$REPLY" != "R" ] ># If REPLY not NUL and not r|R , skip this file & continue loop > then > echo "Leaving $LOCAL as it is" > continue > fi > fi ># Update this local filename to its remote equivalent > mv $LOCAL $REMOTE >done > ># Can remove this temporary file >rm $ISOFILES > >echo ># If there was any anomaly, prompt to continue >if [ $FLAG ] >then > echo "Some ISO filenames were updated" > echo "These are the new local ISO file(names)s: review them" > ls -1 >elif [ $ERR ] >then > echo "There were some duplicate extensions" >else ># Remote = local filenames. > echo "The local & remote ISO filenames are consistent." >fi > ># Offer last chance to quit before rsync-ing ># Allow the user to continue explicitly, otherwise quit >echo -n "OK to continue? <Enter> if yes, anything else to quit: " >read REPLY ># $REPLY needs to be quoted to cope with a null return >if [ "$REPLY" != "" ] ; then exit ; fi > ># At last do the actual rsync update of the selected ISO directory ># The trailing source / = *contents* of the directory ># and destination . = *this* directory, are vital >echo >echo -e "Synchronising this local ISO directory with \n $ISODIR" >echo This can take hours, but can be re-started efficiently if it breaks. >echo >echo `date +%T` rsync starting >echo "rsync -avHP rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ ." >rsync -avHP rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ . >echo `date +%T` rsync finished > ># Do checksums, md5 & sha1 >echo >echo Doing checksums - patience. > ># add additional checksum types to the following line, plus an equivalent case >for CKS in md5 >do > if [ -f *.$CKS ] > then ># A checksum file exists > echo -n "$CKS: " ># The command outputs OK or failure msgs; *.CKS is linked to the *.iso file ># If it is not OK, ouptut the checksum file for info > case "$CKS" in > md5) > md5sum -c *.md5 > ;; ># sha512) ># sha512sum -c *.sha1 ># ;; > esac ># Check for bad result code = checksum failure > if [ $? -ne 0 ] > then > echo -e "\a Supplied $CKS checksum = `cat *.$CKS | awk '{print $1}'`" > fi > else ># Expected checksum file missing > echo -e "\a Warning! No $CKS checksum file" > fi >done > >echo >echo "You have just rsynced this directory $HERE with" >echo " $PHASE/$ISODIR" > >if [ -f DATE.txt ] >then > echo -n " DATE.txt = " > cat DATE.txt >else > echo -e "\a Warning! No DATE.txt file" >fi >echo The essential timestamp is: >ls -l *.iso | awk '{print $9, $6, $7, $8}' > >echo ># If a different directory parameter was given, return to where we were called from >if [ $DIR ] >then cd $FROM ; echo Returned to `pwd` ; fi >
#!/bin/bash # An attempt at a generic ISO rsync update script which copes with file re-naming # onesync -h | -i | DIR # -h Ouput command help # -i Just list remote ISO directory # DIR The local ISO directory to synchronise # . for the current directory # If it does not exist, offer to create it # DEFINE THIS AS NECESSARY & WHEN IT CHANGES RSYNC_PASSWORD="munged" ### <<< UPDATE THIS <<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<!!!!!!!!!!!!!!!! export RSYNC_PASSWORD # THE FOLLOWING ITEMS *COULD* CHANGE but are normally constant # rsync user USER="isoqa" # ISOs host/source directory SOURCE="bcd.mageia.org/isos" USAGE="Usage: onesync -h | -i | DIR" # Check that we have just 1 parameter if [ $# -ne 1 ] then echo $USAGE ; exit ; fi # http://alvinalexander.com/linux-unix/shell-script-command-line-arguments-linux-unix # Look for -h help or -i info option while getopts :hi OPT do # >>>>>>>> case $OPT in # -h for Help, output that and exit h) echo $USAGE echo -e "-h \t Ouput this help" echo -e "-i \t List a remote directory" echo -e "DIR \t The local ISO directory to synchronise" echo -e "\t '.' for the current directory" echo -e "\t If the directory does not exist, it will be optionally created" echo echo -e "\t Examples:" echo onesync -h echo " shows this help text" echo echo onesync -i echo " enables listing a chosen remote ISO" echo echo onesync . echo " synchronises the [ISO] directory it is issued from" echo echo "onesync Mageia/iso/Classic64 [The directory exists]" echo " synchronises the existing [ISO] directory Mageia/iso/Classic64/" echo echo "onesync Mageia/iso/Classic64 [The directory does not yet exist]" echo " optionally creates & primes the [ISO] directory Mageia/iso/Classic64/" echo exit ;; # -i for just remote directory listing simply set INFO flag i) INFO=1 ;; *) echo Invalid parameter exit esac # <<<<<<<< done echo # If called just for info, skip most parts & exit early if [ $INFO ] then echo "This script lists the contents of a chosen ISO on the server" else # Real synchronisation # >>>>>>>> This part skipped for info-only; should all be indented echo "This script primes or updates a single ISO directory" echo # Save where we are called from, to return to at the end if necessary FROM=`pwd` # Get the local directory parameter. '.' means *this* directory if [ $1 != '.' ] then DIR=$1 # Check the directory exists; if not, offer to create it if [ ! -d $1 ] then echo The specified directory $1 does not exist. echo -n "<Enter> to create it, anything else to quit: " read REPLY # $REPLY needs to be quoted to recognise a null return # Continue only if asked to do so, else quit if [ "$REPLY" != "" ] ; then exit ; fi # Create the local ISO directory mkdir $1 fi # Go to the given directory (if it was not '.') cd $DIR fi # Temporary file to hold list of files in the ISO directory ISOFILES=/tmp/ISOfiles`date +%H%M%S` # Confirm that we are in the correct local directory (which can have any name) HERE=`pwd` echo "You should now be in the local ISO directory you want to update:" echo -e "$HERE" # $5=size without commas, $6=coerced yyyy/mm/dd, $7=coerced hh:mm, $8=filename ls -l --time-style='+%Y/%m/%d %H:%M' | awk 'NR>1 {printf "%10s %s %s %s\n", $5, $6, $7, $8}' echo -n "If this is correct, <Enter> to continue; anything else to quit: " read REPLY # $REPLY needs to be quoted to recognise a null return # Continue only if asked to do so, else quit if [ "$REPLY" != "" ] ; then exit ; fi # <<<<<<<< End of part skipped for info-only; should have been indented fi # Start of part shared by info echo # First test the rsync password. Grep should return 0 for 'found' rsync --list-only rsync://${USER}@$SOURCE/ 2>&1 | grep 'auth failed' if [ $? -eq 0 ] then echo The current script rsync password \'$RSYNC_PASSWORD\' seems to be wrong. echo Please update it in the onesync script exit else echo The current script rsync password \'$RSYNC_PASSWORD\' seems OK fi # Read current release umbrella directories echo echo "These are the current releases; choose the one you want:" select PHASE in `rsync --list-only rsync://${USER}@$SOURCE/ | awk 'NR>1 {print $5}'` do # A null reply automatically repeats the prompt # Otherwise, check for out-of-range reply, PHASE = NUL; abort if so. if [ "$PHASE" == "" ] ; then exit ; fi echo -e "\t Chosen release is: $PHASE" break done # In case they responded ^D (EOF) which exits the do loop above to here if [ "$PHASE" == "" ] ; then exit ; fi # Read current ISO directories for chosen release echo "These are its current ISOs; choose CAREFULLY:" select ISODIR in `rsync --list-only rsync://${USER}@$SOURCE/$PHASE/ | awk 'NR>1 {print $5}'` do # A null reply automatically repeats the prompt # Otherwise, check for out-of-range reply, ISODIR = NUL; abort if so. if [ "$ISODIR" == "" ] ; then exit ; fi echo -e "\t Chosen ISO is: $ISODIR" echo Its current content: break done # In case they responded ^D (EOF) which exits the do loop above to here if [ "$ISODIR" == "" ] ; then exit ; fi # Display for info the remote ISO directory contents (skip directory . line) # The tr removes commas (from and only in file size) # $2=size without commas, $3=yyyy/mm/dd, $4=hh:mm:ss cut seconds, $5=filename rsync --list-only rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ \ | tr -d ',' \ | awk 'NR>1 {printf "%10s %s %.5s %s\n", $2, $3, $4, $5}' # End of part shared by info; just exit if [ $INFO ] ; then echo; exit; fi echo -n "Does it look OK? <Enter> to confirm, anything else to quit: " read REPLY # $REPLY needs to be quoted to recognise a null return # Continue only if asked to do so, else quit if [ "$REPLY" != "" ] ; then exit ; fi # Record filenames in the chosen ISO directory to compare with their local names rsync --list-only rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ \ | awk 'NR>1 {print $5}' > $ISOFILES echo echo "Each filename in the remote directory is checked against its" echo "equivalent (based on its extension) in this local one." # THE FILENAME CONSISTENCY ALGORITHM REQUIRES ONLY 1 FILE OF EACH EXTENSION echo If a discrepancy is found, you will be asked to confirm renaming the local copy. echo This is to cope with release filename changes. echo No local equivalent implies a virgin download. echo # Overall record of any local filename changes # FLAG Some re-naming to do # ALL Rename all local files # ERR >1 file of same extension # Work through the list of remote filenames for F in `cat $ISOFILES` do # The remote filename REMOTE=$F # If the remote filename already exists locally, skip it if [ -f $REMOTE ] ; then echo $REMOTE will be updated ; continue ; fi # We did not find an exact match; find the current equivalent local file based on the extension # BASE=${F%.*} # Its extension EXTN=${F##*.} # See if just 1 local file of same extension exists to rename COUNT=`ls *.$EXTN 2>/dev/null | wc -l` # If no equivalent local file, none to rename, it will be downloaded if [ $COUNT -eq 0 ] ; then echo $REMOTE will be downloaded ; continue ; fi # The equivalent local filename(s) LOCAL=`ls *.$EXTN` # If there is >1 filename of this extension, complain & skip them if [ $COUNT -gt 1 ] then ERR=1 echo There is more than one local file of type .$EXTN echo -e "The following local files are NOT renamed: \n $LOCAL" continue fi # We have a filename which can & should be brought up-to-date # Rename the local file to be the same as its remote (extension) equivalent # Only query them if not already asked to do all files. if [ $ALL ] # They want all renamed without confirmation then echo -e "Renaming for update local file \n $LOCAL to \n $REMOTE" else # Ask their opinion FLAG=1 echo echo -e "About to rename for update local file \n $LOCAL \t to \n $REMOTE" echo "<Enter> alone to rename ALL files without asking again," echo " r|R to rename just THIS file," echo -n " anything else to leave this filename as it is: " read REPLY # Rename only if asked to do so, otherwise skip this file # $REPLY needs to be quoted to recognise a null return if [ "$REPLY" == "" ] # Re-naming without further confirmation then ALL=1 elif [ "$REPLY" != "r" -a "$REPLY" != "R" ] # If REPLY not NUL and not r|R , skip this file & continue loop then echo "Leaving $LOCAL as it is" continue fi fi # Update this local filename to its remote equivalent mv $LOCAL $REMOTE done # Can remove this temporary file rm $ISOFILES echo # If there was any anomaly, prompt to continue if [ $FLAG ] then echo "Some ISO filenames were updated" echo "These are the new local ISO file(names)s: review them" ls -1 elif [ $ERR ] then echo "There were some duplicate extensions" else # Remote = local filenames. echo "The local & remote ISO filenames are consistent." fi # Offer last chance to quit before rsync-ing # Allow the user to continue explicitly, otherwise quit echo -n "OK to continue? <Enter> if yes, anything else to quit: " read REPLY # $REPLY needs to be quoted to cope with a null return if [ "$REPLY" != "" ] ; then exit ; fi # At last do the actual rsync update of the selected ISO directory # The trailing source / = *contents* of the directory # and destination . = *this* directory, are vital echo echo -e "Synchronising this local ISO directory with \n $ISODIR" echo This can take hours, but can be re-started efficiently if it breaks. echo echo `date +%T` rsync starting echo "rsync -avHP rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ ." rsync -avHP rsync://${USER}@$SOURCE/$PHASE/$ISODIR/ . echo `date +%T` rsync finished # Do checksums, md5 & sha1 echo echo Doing checksums - patience. # add additional checksum types to the following line, plus an equivalent case for CKS in md5 do if [ -f *.$CKS ] then # A checksum file exists echo -n "$CKS: " # The command outputs OK or failure msgs; *.CKS is linked to the *.iso file # If it is not OK, ouptut the checksum file for info case "$CKS" in md5) md5sum -c *.md5 ;; # sha512) # sha512sum -c *.sha1 # ;; esac # Check for bad result code = checksum failure if [ $? -ne 0 ] then echo -e "\a Supplied $CKS checksum = `cat *.$CKS | awk '{print $1}'`" fi else # Expected checksum file missing echo -e "\a Warning! No $CKS checksum file" fi done echo echo "You have just rsynced this directory $HERE with" echo " $PHASE/$ISODIR" if [ -f DATE.txt ] then echo -n " DATE.txt = " cat DATE.txt else echo -e "\a Warning! No DATE.txt file" fi echo The essential timestamp is: ls -l *.iso | awk '{print $9, $6, $7, $8}' echo # If a different directory parameter was given, return to where we were called from if [ $DIR ] then cd $FROM ; echo Returned to `pwd` ; fi
View Attachment As Raw
Actions:
View
Attachments on
bug 26745
: 11685