From 4cf2379d3c885d6dd7f0789e15e49610e2e0c7a9 Mon Sep 17 00:00:00 2001
From: David Beaumont <dbeaumont@google.com>
Date: Tue, 18 Aug 2020 13:36:54 +0000
Subject: [PATCH] ICU-21066 Improvements in Ant build files

See #1224
---
 icu4c/source/data/cldr-icu-readme.txt     | 13 ++---
 tools/cldr/build.xml                      | 65 +++++++++++++++++++++++
 tools/cldr/cldr-to-icu/README.txt         | 32 +++++------
 tools/cldr/cldr-to-icu/build-icu-data.xml | 52 ++++++++++++++----
 4 files changed, 126 insertions(+), 36 deletions(-)
 create mode 100644 tools/cldr/build.xml

diff --git a/icu4c/source/data/cldr-icu-readme.txt b/icu4c/source/data/cldr-icu-readme.txt
index fed367e34f4..701ba981f54 100644
--- a/icu4c/source/data/cldr-icu-readme.txt
+++ b/icu4c/source/data/cldr-icu-readme.txt
@@ -55,12 +55,9 @@
 #
 # b) CLDR-related variables
 #
-# CLDR_DIR:      For most of the process, this is the path to the to root of
-#                standard CLDR sources, below which are the common and
-#                tools directories. For running LdmlConverter, this needs to be
-#                temporarily reset to the parallel root for the production data,
-#                corresponding to $CLDR_TMP_DIR/production (see description of
-#                CLDR_TMP_DIR below).
+# CLDR_DIR:      This is the path to the to root of standard CLDR sources, below
+#                which are the common and tools directories.
+#
 # CLDR_CLASSES:  Path to the CLDR Tools classes directory. If not set, defaults
 #                to $CLDR_DIR/tools/java/classes
 #
@@ -177,8 +174,6 @@ ant proddata 2>&1 | tee /tmp/cldr-newData-proddataLog.txt
 # output (so do not assume nothing is happening). Keep a log so you can investigate
 # anything that looks suspicious.
 #
-# This also requires temporarily redefining CLDR_DIR.
-#
 # Note that "ant clean" should not be run before this. The build-icu-data.xml process
 # will automatically run its own "clean" step to delete files it cannot determine to
 # be ones that it would generate, except for pasts listed in <retain> elements such as
@@ -188,7 +183,7 @@ ant proddata 2>&1 | tee /tmp/cldr-newData-proddataLog.txt
 # build-icu-data.xml file, such as adding new locales etc.
 
 cd $TOOLS_ROOT/cldr/cldr-to-icu
-CLDR_DIR=$CLDR_TMP_DIR/production ant -f build-icu-data.xml | tee /tmp/cldr-newData-builddataLog.txt
+ant -f build-icu-data.xml -DcldrDataDir="$CLDR_TMP_DIR/production" | tee /tmp/cldr-newData-builddataLog.txt
 
 # 5. Check which data files have modifications, which have been added or removed
 # (if there are no changes, you may not need to proceed further). Make sure the
diff --git a/tools/cldr/build.xml b/tools/cldr/build.xml
new file mode 100644
index 00000000000..76aaec7af74
--- /dev/null
+++ b/tools/cldr/build.xml
@@ -0,0 +1,65 @@
+<!-- © 2020 and later: Unicode, Inc. and others.
+     License & terms of use: http://www.unicode.org/copyright.html -->
+
+<!-- This build file is intended to become the single mechanism for working with CLDR
+     code and data when building ICU data.
+     
+     Eventually it will encompass:
+     * Building ICU data form CLDR data via cldr-to-icu.
+     * Building the CLDR libraries needed to support ICU data conversion.
+     * Copying CLDR test data for ICU regression tests.
+
+     It's not complete yet, so for now follow the instructions in:
+       icu4c/source/data/cldr-icu-readme.txt
+     -->
+
+<!-- Using the directory of this build.xml file as ${basedir}. -->
+<project name="CLDR" default="install" basedir=".">
+
+    <!-- Installs prerequisite data from CLDR (test data and CLDR libraries).
+         This target will not rebuild the ICU data however (since that's a
+         lengthy process). -->
+    <target name="install" depends="clean, copy-cldr-testdata, install-cldr-libs"/>
+
+    <!-- Cleans CLDR dependencies (testdata, CLDR libraries). This target does
+         not delete any generated ICU data. -->
+    <target name="clean" depends="clean-cldr-testdata"/>
+
+    <!-- Initialize any properties not already set on the command line. -->
+    <target name="init-args">
+        <property environment="env"/>
+        <!-- Inherit properties from environment variable unless specified. As usual
+             with Ant, this is messier than it should be. All we are saying here is:
+             "Use the property if explicitly set, otherwise use the environment variable."
+             We cannot just set the property to the environment variable, since expansion
+             fails for non existant properties, and you are left with a literal value of
+             "${env.CLDR_DIR}". -->
+        <condition property="cldrDir" value="${env.CLDR_DIR}">
+            <isset property="env.CLDR_DIR"/>
+        </condition>
+        <fail unless="cldrDir"
+             message="Set the CLDR_DIR environment variable (or cldrDir property) to the top level CLDR source directory (containing 'common')."/>
+    </target>
+
+    <!-- TODO: Add properties and targets for copying test data. -->
+
+    <!-- Builds the ICU data, using the Ant build file in the cldr-to-icu directory and passing.
+         through any specified arguments for controlling the build. If you need more control when
+         building ICU data (such as incrementally building parts of the data), you should use the
+         build-icu-data.xml file directly. -->
+    <target name="build-icu-data">
+        <ant dir="cldr-to-icu" antfile="build-icu-data.xml" target="all" inheritAll="true"/>
+    </target>
+
+    <!-- Deletes generated ICU data by invoking "clean" in cldr-to-icu/build-icu-data.xml -->
+    <target name="clean-icu-data">
+        <ant dir="cldr-to-icu" antfile="build-icu-data.xml" target="clean" inheritAll="true"/>
+    </target>
+
+    <!-- Installs the CLDR library dependencies needed for building ICU data. -->
+    <target name="install-cldr-libs" depends="init-args">
+        <exec dir="lib" executable="install-cldr-jars.sh" resolveexecutable="true" failonerror="true">
+            <arg line="${cldrDir}"/>
+        </exec>
+    </target>
+</project>
diff --git a/tools/cldr/cldr-to-icu/README.txt b/tools/cldr/cldr-to-icu/README.txt
index 4bde0a4487a..2329dd4a2b4 100644
--- a/tools/cldr/cldr-to-icu/README.txt
+++ b/tools/cldr/cldr-to-icu/README.txt
@@ -16,23 +16,23 @@ Requirements
 Important directories
 ---------------------
 
-ICU_ROOT = The top-level directory for this ICU project installation.
+TOOLS_ROOT: Path to root of ICU tools directory, below which are (e.g.) the
+            cldr/ and unicodetools/ directories.
 
-CLDR_ROOT = The top-level directory for the CLDR project, containing CLDR
-            code and non-production data. Usually obtained from:
-            https://github.com/unicode-org/cldr
+CLDR_DIR:   This is the path to the to root of standard CLDR sources, below
+            which are the common/ and tools/ directories.
 
-CLDR_DIR  = The top-level directory for the CLDR production data (typically
-            the "production" directory in the staging repository). Usually
-            obtained from:
-            https://github.com/unicode-org/cldr-staging/tree/master/production
+CLDR_DATA_DIR: The top-level directory for the CLDR production data (typically
+               the "production" directory in the staging repository). Usually
+               generated locally or obtained from:
+               https://github.com/unicode-org/cldr-staging/tree/master/production
 
 In Posix systems, it's best to set these as exported shell variables, and any
 following instructions assume they have been set accordingly:
 
-$ export ICU_ROOT=/path/to/icu
-$ export CLDR_ROOT=/path/to/cldr
-$ export CLDR_DIR=/path/to/cldr-staging/production
+$ export TOOLS_ROOT=/path/to/icu/tools
+$ export CLDR_DIR=/path/to/cldr
+$ export CLDR_DATA_DIR=/path/to/cldr-staging/production
 
 Note that you should not attempt to use data from the CLDR project directory
 (where the CLDR API code exists) for conversion into ICU data. The process now
@@ -51,17 +51,17 @@ based system, this should be as simple as:
 $ sudo apt-get install maven ant
 
 You must also install an additional CLDR JAR file the local Maven repository at
-$ICU_ROOT/tools/cldr/lib (see the README.txt in that directory for more
+$TOOLS_ROOT/cldr/lib (see the README.txt in that directory for more
 information).
 
-$ cd "$ICU_ROOT/tools/cldr/lib"
-$ ./install-cldr-jars.sh "$CLDR_ROOT"
+$ cd "$TOOLS_ROOT/cldr/lib"
+$ ./install-cldr-jars.sh "$CLDR_DIR"
 
 
 Generating all ICU data
 -----------------------
 
-$ cd "$ICU_ROOT/tools/cldr/cldr-to-icu"
+$ cd "$TOOLS_ROOT/cldr/cldr-to-icu"
 $ ant -f build-icu-data.xml
 
 
@@ -92,7 +92,7 @@ See build-icu-data.xml for documentation of all options and additional customiza
 Running unit tests
 ------------------
 
-$ mvn test -DCLDR_DIR="$CLDR_DIR"
+$ mvn test -DCLDR_DIR="$CLDR_DATA_DIR"
 
 
 Importing and running from an IDE
diff --git a/tools/cldr/cldr-to-icu/build-icu-data.xml b/tools/cldr/cldr-to-icu/build-icu-data.xml
index 244191ae96a..2d36103a8d5 100644
--- a/tools/cldr/cldr-to-icu/build-icu-data.xml
+++ b/tools/cldr/cldr-to-icu/build-icu-data.xml
@@ -20,18 +20,23 @@
     <!-- Initialize the properties which were not already set on the command line. -->
     <target name="init-args">
         <property environment="env"/>
-        <condition property="hascldrdir" >
-            <isset property="env.CLDR_DIR" />
+        <!-- Inherit properties from environment variable unless specified. As usual
+             with Ant, this is messier than it should be. All we are saying here is:
+             "Use the property if explicitly set, otherwise use the environment variable."
+             We cannot just set the property to the environment variable, since expansion
+             fails for non existant properties, and you are left with a literal value of
+             "${env.CLDR_DATA_DIR}". -->
+        <condition property="cldrDataDir" value="${env.CLDR_DATA_DIR}">
+            <isset property="env.CLDR_DATA_DIR"/>
         </condition>
-        <fail unless="hascldrdir"
-              message="Please set the CLDR_DIR environment variable to the top level CLDR source dir (containing 'common')."/>
+        <fail unless="cldrDataDir"
+              message="Set the CLDR_DATA_DIR environment variable (or cldrDataDir property) to the CLDR data directory (typically ending in '/production')"/>
 
-        <!-- The base directory of the CLDR release from which CLDR data is obtained. For
-             legacy reasons, this must also match an environment variable called CLDR_DIR,
-             which is read here, so it is best to set this via the environment variable for
-             now. -->
-        <!-- TODO: Update this when the CLDR_DIR environment variable is no longer needed. -->
-        <property name="cldrDir" value="${env.CLDR_DIR}"/>
+        <!-- Ant does not inherit this from the user's environment (and it can matter).
+             This is only needed because we have to "exec" a new Ant task below. -->
+        <condition property="javaHome" value="${env.JAVA_HOME}">
+            <isset property="env.JAVA_HOME"/>
+        </condition>
 
         <!-- The output directory into which to write the converted ICU data. By default
              this will overwrite (without deletion) the ICU data files in this ICU release,
@@ -88,9 +93,34 @@
         </exec>
     </target>
 
+    <!-- Somewhat hacky wrapper target which invokes the real conversion task.
+         This is done so we can set the environment variable of the new process and
+         effectively overwrite the CLDR_DIR value. If ever the CLDR library doesn't
+         need to use CLDR_DIR at runtime to find the production data, this can all be
+         removed. -->
+    <target name="convert" depends="init-args, prepare-jar">
+        <exec executable="ant" searchpath="true" failonerror="true">
+            <!-- The CLDR library wants CLDR_DIR set, to the data directory. -->
+            <env key="CLDR_DIR" value="${cldrDataDir}" />
+            <!-- Force inherit JAVA_HOME (this can be important). -->
+            <env key="JAVA_HOME" value="${javaHome}" />
+            <!-- Initial Ant command line with all the "interesting" bit in. -->
+            <arg line="-f build-icu-data.xml convert-impl -DcldrDir=${cldrDataDir}"/>
+            <!-- List all properties in the "convert-impl" task (except cldrDir). -->
+            <arg value="-DoutDir=${outDir}"/>
+            <arg value="-DspecialsDir=${specialsDir}"/>
+            <arg value="-DoutputTypes=${outputTypes}"/>
+            <arg value="-DcldrVersion=${cldrVersion}"/>
+            <arg value="-DminDraftStatus=${minDraftStatus}"/>
+            <arg value="-DlocaleIdFilter=${localeIdFilter}"/>
+            <arg value="-DincludePseudoLocales=${includePseudoLocales}"/>
+            <arg value="-DemitReport=${emitReport}"/>
+        </exec>
+    </target>
+
     <!-- Do the actual CLDR data conversion, based on the command line arguments, built in
          default properties and the configuration in the "<convert>" element below. -->
-    <target name="convert" depends="init-args, prepare-jar">
+    <target name="convert-impl">
         <taskdef name="convert" classname="org.unicode.icu.tool.cldrtoicu.ant.ConvertIcuDataTask">
             <classpath>
                 <pathelement path="target/cldr-to-icu-1.0-SNAPSHOT-jar-with-dependencies.jar"/>
-- 
2.40.0