From: Ben Soares <b.soares@dundee.ac.uk>
Date: Tue, 1 Nov 2022 11:16:06 +0000 (+0000)
Subject: JAL-4064 Download JRE/JDKs from Azul if not os/arch combination not available from... 
X-Git-Tag: Release_2_11_3_0~23^2~4^2~16^2~5
X-Git-Url: http://source.jalview.org/gitweb/?a=commitdiff_plain;h=80458cd593a992843a8eee1651ca489b6bfdfbb9;p=jalview.git

JAL-4064 Download JRE/JDKs from Azul if not os/arch combination not available from Adoptium
---

diff --git a/utils/download_jdks.sh b/utils/download_jdks.sh
index cf8634a..b6e8379 100755
--- a/utils/download_jdks.sh
+++ b/utils/download_jdks.sh
@@ -15,26 +15,64 @@
 ### Edited to use adoptium domain to gain access to Java 17 (LTS) versions.
 
 BASE=https://api.adoptium.net/v3/binary/latest
+ZULU_BASE=https://cdn.azul.com/zulu/bin
 RELEASE_TYPE=ga
 JVM_IMPL=hotspot
 HEAP_SIZE=normal
 VENDOR=eclipse
 IMAGE_TYPE=jdk
+TAR=tar
+ZIP=zip
+UNZIP=unzip
+
+STRIP_MAC_APP_BUNDLING=false
+# archives not needed for JDKs
+CREATE_ARCHIVES=""
+# need zip with top-level jre dir for getdown updates. need tgz without top-level jre dir for install4j bundling
 
 RM=/bin/rm
 
 # unzip-strip from https://superuser.com/questions/518347/equivalent-to-tars-strip-components-1-in-unzip
-unzip-strip() (
+unzip-strip() {
   local zip=$1
   local dest=${2:-.}
-  local temp=$(mktemp -d) && unzip -qq -d "$temp" "$zip" && mkdir -p "$dest" &&
+  local temp=$(mktemp -d) && $UNZIP -qq -d "$temp" "$zip" && mkdir -p "$dest" &&
   shopt -s dotglob && local f=("$temp"/*) &&
   if (( ${#f[@]} == 1 )) && [[ -d "${f[0]}" ]] ; then
     mv "$temp"/*/* "$dest"
   else
     mv "$temp"/* "$dest"
   fi && rmdir "$temp"/* "$temp"
-)
+}
+
+dl_zulu() {
+  local OS="$1"
+  local ARCH="$2"
+  local VERSION="$3"
+  local TARFILE="$4"
+  declare -A osmap
+  osmap[mac]=macosx
+  osmap[windows]=win
+  osmap[linux]=linux
+  ZOS="${osmap[$OS]}"
+  echo "- Looking for download from Azul"
+  LATEST_DL_URL_FILE=$(wget -q -O - "${ZULU_BASE}/" | perl -n -e 'm/<a\b[^>]*href="(([^"]*\/)?zulu[^"]*-ca-'"${IMAGE_TYPE}""${VERSION}"'\.[^"]*-'"${ZOS}"'_'"${ARCH}"'.tar.gz)"[^"]*>/ && print "$1\n";' | tail -1)
+  local URL="${ZULU_BASE}/${LATEST_DL_URL_FILE}"
+  if [ -z "${LATEST_DL_URL_FILE}" ]; then
+    echo "- No ${IMAGE_TYPE}-${FEATURE_VERSION} download for ${OS}-${ARCH} '${URL}' found at Azul"
+    return 1
+  fi
+  echo "- Found at Azul. Downloading '${URL}'"
+  wget -q -O "${TARFILE}" "${URL}" "${TARFILE}"
+  echo RETURN=$?
+  if [ "$?" != 0 ]; then
+    echo "- Download from Azul failed"
+    return 1
+  fi
+  return 0
+}
+
+declare -A DOWNLOAD_SUMMARY
 
 for FEATURE_VERSION in 8 11 17; do
   for OS_ARCH in mac:x64 mac:aarch64 windows:x64 linux:x64 linux:arm linux:aarch64; do
@@ -42,13 +80,30 @@ for FEATURE_VERSION in 8 11 17; do
     ARCH=${OS_ARCH#*:}
     NAME="${IMAGE_TYPE}-${FEATURE_VERSION}-${OS}-${ARCH}"
     TARFILE="${NAME}.tgz"
+    DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="None"
+    STRIP_COMPONENTS=1
+    MAC_STRIP_COMPONENTS=3
     echo "* Downloading ${TARFILE}"
     URL="${BASE}/${FEATURE_VERSION}/${RELEASE_TYPE}/${OS}/${ARCH}/${IMAGE_TYPE}/${JVM_IMPL}/${HEAP_SIZE}/${VENDOR}"
     wget -q -O "${TARFILE}" "${URL}"
     if [ "$?" != 0 ]; then
-      echo "- No ${IMAGE_TYPE}-${FEATURE_VERSION} download for ${OS}-${ARCH} '${URL}'"
+      echo "- No ${IMAGE_TYPE}-${FEATURE_VERSION} download for ${OS}-${ARCH} '${URL}' at Adoptium"
       $RM -f "${TARFILE}"
-      continue;
+
+      # Try Azul Zulu (not an API, a bit messier, but has Java 8 JRE for mac:aarch64
+      dl_zulu "${OS}" "${ARCH}" "${FEATURE_VERSION}" "${TARFILE}"
+
+      if [ "$?" != 0 ]; then
+        DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="None"
+        continue;
+      fi
+      STRIP_COMPONENTS=2
+      MAC_STRIP_COMPONENTS=4
+      DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="Azul"
+      echo "Set ${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}=Azul"
+    else
+      DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="Adoptium"
+      echo "Set ${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}=Adoptium"
     fi
     echo "Unpacking ${TARFILE}"
     JREDIR="${NAME}/${IMAGE_TYPE}"
@@ -60,14 +115,58 @@ for FEATURE_VERSION in 8 11 17; do
       RET=$?
     else
       echo "using tar"
-      tar --strip-components=1 -C "${JREDIR}" -zxf "${TARFILE}"
-      RET=$?
+      if [ x$OS = xmac -a x$STRIP_MAC_APP_BUNDLING = xtrue ]; then
+        echo "Running $TAR --strip-components=\"${MAC_STRIP_COMPONENTS}\" -C \"${JREDIR}\" -zxf \"${TARFILE}\" \"*/Contents/Home\""
+        $TAR --strip-components="${MAC_STRIP_COMPONENTS}" -C "${JREDIR}" -zxf "${TARFILE}" "*/Contents/Home"
+        RET=$?
+      else
+        $TAR --strip-components="${STRIP_COMPONENTS}" -C "${JREDIR}" -zxf "${TARFILE}"
+        RET=$?
+      fi
     fi
     if [ "$RET" != 0 ]; then
       echo "Error unpacking ${TARFILE}"
       exit 1
     fi
     $RM "${TARFILE}"
+    if [ \! -z "$CREATE_ARCHIVES" ]; then
+      for CREATEARCHIVE in ${CREATE_ARCHIVES}; do
+        ARCHIVEDIR=$CREATEARCHIVE
+        case $CREATEARCHIVE in
+          zip)
+            EXT=${CREATEARCHIVE}
+            echo "Creating ${NAME}.${EXT} for getdown updates"
+            [ \! -d ${ARCHIVEDIR} ] && mkdir -p "${ARCHIVEDIR}"
+            ABSARCHIVEDIR="${PWD}/$ARCHIVEDIR"
+            ZIPFILE="${ABSARCHIVEDIR}/${NAME}.${CREATEARCHIVE}"
+            [ -e "${ZIPFILE}" ] && $RM "${ZIPFILE}"
+            cd ${NAME}
+            $ZIP -X -r "${ZIPFILE}" "${IMAGE_TYPE}"
+            cd -
+            ;;
+          tgz)
+            EXT=tar.gz
+            echo "Creating ${NAME}.${EXT} for install4j bundling"
+            [ \! -d ${ARCHIVEDIR} ] && mkdir -p "${ARCHIVEDIR}"
+            $TAR -C "${JREDIR}" -zcf "${ARCHIVEDIR}/${NAME}.${EXT}" .
+            # make symbolic link with _ instead of - for install4j9
+            NEWNAME=${NAME//-/_}
+            echo "Linking from ${NEWNAME}.${EXT} for install4j9"
+            [ -e "${ARCHIVEDIR}/${NEWNAME}.${EXT}" ] && $RM "${ARCHIVEDIR}/${NEWNAME}.${EXT}"
+            ln -s "${NAME}.${EXT}" "${ARCHIVEDIR}/${NEWNAME}.${EXT}"
+            ;;
+          *)
+            echo "Archiving as '${CREATEARCHIVE}' file not supported"
+            ;;
+        esac
+      done
+    fi
   done
 done
 
+echo ""
+echo "Download Summary"
+for OA in "${!DOWNLOAD_SUMMARY[@]}"; do
+  echo "$OA: ${DOWNLOAD_SUMMARY[$OA]}"
+done
+
diff --git a/utils/download_jres.sh b/utils/download_jres.sh
index 26442ea..c1a5a0d 100755
--- a/utils/download_jres.sh
+++ b/utils/download_jres.sh
@@ -15,6 +15,7 @@
 ### Edited to use adoptium domain to gain access to Java 17 (LTS) versions.
 
 BASE=https://api.adoptium.net/v3/binary/latest
+ZULU_BASE=https://cdn.azul.com/zulu/bin
 RELEASE_TYPE=ga
 JVM_IMPL=hotspot
 HEAP_SIZE=normal
@@ -31,7 +32,7 @@ CREATE_ARCHIVES="zip tgz"
 RM=/bin/rm
 
 # unzip-strip from https://superuser.com/questions/518347/equivalent-to-tars-strip-components-1-in-unzip
-unzip-strip() (
+unzip-strip() {
   local zip=$1
   local dest=${2:-.}
   local temp=$(mktemp -d) && $UNZIP -qq -d "$temp" "$zip" && mkdir -p "$dest" &&
@@ -41,7 +42,36 @@ unzip-strip() (
   else
     mv "$temp"/* "$dest"
   fi && rmdir "$temp"/* "$temp"
-)
+}
+
+dl_zulu() {
+  local OS="$1"
+  local ARCH="$2"
+  local VERSION="$3"
+  local TARFILE="$4"
+  declare -A osmap
+  osmap[mac]=macosx
+  osmap[windows]=win
+  osmap[linux]=linux
+  ZOS="${osmap[$OS]}"
+  echo "- Looking for download from Azul"
+  LATEST_DL_URL_FILE=$(wget -q -O - "${ZULU_BASE}/" | perl -n -e 'm/<a\b[^>]*href="(([^"]*\/)?zulu[^"]*-ca-'"${IMAGE_TYPE}""${VERSION}"'\.[^"]*-'"${ZOS}"'_'"${ARCH}"'.tar.gz)"[^"]*>/ && print "$1\n";' | tail -1)
+  local URL="${ZULU_BASE}/${LATEST_DL_URL_FILE}"
+  if [ -z "${LATEST_DL_URL_FILE}" ]; then
+    echo "- No ${IMAGE_TYPE}-${FEATURE_VERSION} download for ${OS}-${ARCH} '${URL}' found at Azul"
+    return 1
+  fi
+  echo "- Found at Azul. Downloading '${URL}'"
+  wget -q -O "${TARFILE}" "${URL}" "${TARFILE}"
+  echo RETURN=$?
+  if [ "$?" != 0 ]; then
+    echo "- Download from Azul failed"
+    return 1
+  fi
+  return 0
+}
+
+declare -A DOWNLOAD_SUMMARY
 
 for FEATURE_VERSION in 8 11 17; do
   for OS_ARCH in mac:x64 mac:aarch64 windows:x64 linux:x64 linux:arm linux:aarch64; do
@@ -49,13 +79,30 @@ for FEATURE_VERSION in 8 11 17; do
     ARCH=${OS_ARCH#*:}
     NAME="${IMAGE_TYPE}-${FEATURE_VERSION}-${OS}-${ARCH}"
     TARFILE="${NAME}.tgz"
+    DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="None"
+    STRIP_COMPONENTS=1
+    MAC_STRIP_COMPONENTS=3
     echo "* Downloading ${TARFILE}"
     URL="${BASE}/${FEATURE_VERSION}/${RELEASE_TYPE}/${OS}/${ARCH}/${IMAGE_TYPE}/${JVM_IMPL}/${HEAP_SIZE}/${VENDOR}"
     wget -q -O "${TARFILE}" "${URL}"
     if [ "$?" != 0 ]; then
-      echo "- No ${IMAGE_TYPE}-${FEATURE_VERSION} download for ${OS}-${ARCH} '${URL}'"
+      echo "- No ${IMAGE_TYPE}-${FEATURE_VERSION} download for ${OS}-${ARCH} '${URL}' at Adoptium"
       $RM -f "${TARFILE}"
-      continue;
+
+      # Try Azul Zulu (not an API, a bit messier, but has Java 8 JRE for mac:aarch64
+      dl_zulu "${OS}" "${ARCH}" "${FEATURE_VERSION}" "${TARFILE}"
+
+      if [ "$?" != 0 ]; then
+        DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="None"
+        continue;
+      fi
+      STRIP_COMPONENTS=2
+      MAC_STRIP_COMPONENTS=4
+      DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="Azul"
+      echo "Set ${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}=Azul"
+    else
+      DOWNLOAD_SUMMARY["${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}"]="Adoptium"
+      echo "Set ${OS_ARCH}-${IMAGE_TYPE}-${FEATURE_VERSION}=Adoptium"
     fi
     echo "Unpacking ${TARFILE}"
     JREDIR="${NAME}/${IMAGE_TYPE}"
@@ -68,10 +115,11 @@ for FEATURE_VERSION in 8 11 17; do
     else
       echo "using tar"
       if [ x$OS = xmac -a x$STRIP_MAC_APP_BUNDLING = xtrue ]; then
-        $TAR --strip-components=3 -C "${JREDIR}" -zxf "${TARFILE}" "*/Contents/Home"
+        echo "Running $TAR --strip-components=\"${MAC_STRIP_COMPONENTS}\" -C \"${JREDIR}\" -zxf \"${TARFILE}\" \"*/Contents/Home\""
+        $TAR --strip-components="${MAC_STRIP_COMPONENTS}" -C "${JREDIR}" -zxf "${TARFILE}" "*/Contents/Home"
         RET=$?
       else
-        $TAR --strip-components=1 -C "${JREDIR}" -zxf "${TARFILE}"
+        $TAR --strip-components="${STRIP_COMPONENTS}" -C "${JREDIR}" -zxf "${TARFILE}"
         RET=$?
       fi
     fi
@@ -115,3 +163,9 @@ for FEATURE_VERSION in 8 11 17; do
   done
 done
 
+echo ""
+echo "Download Summary"
+for OA in "${!DOWNLOAD_SUMMARY[@]}"; do
+  echo "$OA: ${DOWNLOAD_SUMMARY[$OA]}"
+done
+