ICU-10923 Adding file replacement mechanism to buildtool.

diff --git a/icu4c/source/configure b/icu4c/source/configure
index 91c3513743baca5026a43a31523d7f1338752125..274f361434928b53ddd7a48de7fc02d586b50907 100755 (executable)
--- a/icu4c/source/configure
+++ b/icu4c/source/configure
@@ -9131,7 +9131,7 @@ else
     PYTHONPATH="$srcdir/data" $PYTHON -m buildtool \
         --mode gnumake \
         --seqmode parallel \
-        --in_dir "$srcdir/data" \
+        --src_dir "$srcdir/data" \
         --filter_file "$ICU_DATA_FILTER_FILE" \
         > data/rules.mk
     if test "$?" != "0"; then
@@ -9141,7 +9141,7 @@ else
     PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON -m buildtool \
         --mode gnumake \
         --seqmode parallel \
-        --in_dir "$srcdir/test/testdata" \
+        --src_dir "$srcdir/test/testdata" \
         > test/testdata/rules.mk
     if test "$?" != "0"; then
         as_fn_error $? "Python failed to run; see above error." "$LINENO" 5

diff --git a/icu4c/source/configure.ac b/icu4c/source/configure.ac
index e5feb4cde5be4fb5295ae352b0ff00f199b83d92..ef3e72fec4896ff7e6ead7c1e25a87c4ad7b87bc 100644 (file)
--- a/icu4c/source/configure.ac
+++ b/icu4c/source/configure.ac
@@ -1395,7 +1395,7 @@ else
     PYTHONPATH="$srcdir/data" $PYTHON -m buildtool \
         --mode gnumake \
         --seqmode parallel \
-        --in_dir "$srcdir/data" \
+        --src_dir "$srcdir/data" \
         --filter_file "$ICU_DATA_FILTER_FILE" \
         > data/rules.mk
     if test "$?" != "0"; then
@@ -1405,7 +1405,7 @@ else
     PYTHONPATH="$srcdir/test/testdata:$srcdir/data" $PYTHON -m buildtool \
         --mode gnumake \
         --seqmode parallel \
-        --in_dir "$srcdir/test/testdata" \
+        --src_dir "$srcdir/test/testdata" \
         > test/testdata/rules.mk
     if test "$?" != "0"; then
         AC_MSG_ERROR(Python failed to run; see above error.)

diff --git a/icu4c/source/data/BUILDRULES.py b/icu4c/source/data/BUILDRULES.py
index 12e208911871465638a5f9b26bf3fcc7d41c2530..bdcd781418bd2257f02e5d92f8b93c48753fc4de 100644 (file)
--- a/icu4c/source/data/BUILDRULES.py
+++ b/icu4c/source/data/BUILDRULES.py
@@ -18,7 +18,7 @@ def generate(config, glob, common_vars):
     requests = []
 
     if len(glob("misc/*")) == 0:
-        print("Error: Cannot find data directory; please specify --in_dir", file=sys.stderr)
+        print("Error: Cannot find data directory; please specify --src_dir", file=sys.stderr)
         exit(1)
 
     requests += generate_cnvalias(config, glob, common_vars)
@@ -101,7 +101,7 @@ def generate(config, glob, common_vars):
         # Depends on timezoneTypes.res and keyTypeData.res.
         # TODO: We should not need this dependency to build collation.
         # TODO: Bake keyTypeData.res into the common library?
-        [DepTarget("coll_ucadata"), DepTarget("misc_res")])
+        [DepTarget("coll_ucadata"), DepTarget("misc_res"), InFile("unidata/UCARules.txt")])
 
     requests += generate_tree(config, glob, common_vars,
         "brkitr",
@@ -225,7 +225,7 @@ def generate_stringprep(config, glob, common_vars):
         RepeatedExecutionRequest(
             name = "stringprep",
             category = "stringprep",
-            dep_targets = [],
+            dep_targets = [InFile("unidata/NormalizationCorrections.txt")],
             input_files = input_files,
             output_files = output_files,
             tool = IcuTool("gensprep"),
@@ -398,6 +398,9 @@ def generate_translit(config, glob, common_vars):
         InFile("translit/en.txt"),
         InFile("translit/el.txt")
     ]
+    dep_files = set(InFile(filename) for filename in glob("translit/*.txt"))
+    dep_files -= set(input_files)
+    dep_files = list(dep_files)
     input_basenames = [v.filename[9:] for v in input_files]
     output_files = [
         OutFile("translit/%s.res" % v[:-4])
@@ -407,7 +410,7 @@ def generate_translit(config, glob, common_vars):
         RepeatedOrSingleExecutionRequest(
             name = "translit_res",
             category = "translit",
-            dep_targets = [],
+            dep_targets = dep_files,
             input_files = input_files,
             output_files = output_files,
             tool = IcuTool("genrb"),

diff --git a/icu4c/source/data/buildtool/__init__.py b/icu4c/source/data/buildtool/__init__.py
index 04a8c34cd2b89ddd5429c77797705b1bcf5d243a..be936166e7eafd378546fc5ea9a91f37e9e294b9 100644 (file)
--- a/icu4c/source/data/buildtool/__init__.py
+++ b/icu4c/source/data/buildtool/__init__.py
@@ -3,6 +3,8 @@
 
 from collections import namedtuple
 
+LocalFile = namedtuple("LocalFile", ["dirname", "filename"])
+SrcFile = namedtuple("SrcFile", ["filename"])
 InFile = namedtuple("InFile", ["filename"])
 TmpFile = namedtuple("TmpFile", ["filename"])
 OutFile = namedtuple("OutFile", ["filename"])

diff --git a/icu4c/source/data/buildtool/__main__.py b/icu4c/source/data/buildtool/__main__.py
index 40c33a8f14803324f74833a3c3e1b5d30bb472f9..6bc2ef9ac7dec00c0fc34e0a3ccb55b9f24fc6d2 100644 (file)
--- a/icu4c/source/data/buildtool/__main__.py
+++ b/icu4c/source/data/buildtool/__main__.py
@@ -13,6 +13,7 @@ import sys
 
 from . import *
 from .comment_stripper import CommentStripper
+from .request_types import CopyRequest
 from .renderers import makefile, unix_exec, windows_exec
 from . import filtration, utils
 import BUILDRULES
@@ -67,8 +68,8 @@ arg_group_required.add_argument(
 )
 
 flag_parser.add_argument(
-    "--in_dir",
-    help = "Path to data input folder (icu4c/source/data).",
+    "--src_dir",
+    help = "Path to data source folder (icu4c/source/data).",
     default = "."
 )
 flag_parser.add_argument(
@@ -163,12 +164,52 @@ class Config(object):
             pass
 
 
+def add_copy_input_requests(requests, config, common_vars):
+    files_to_copy = set()
+    for request in requests:
+        for f in request.all_input_files():
+            if isinstance(f, InFile):
+                files_to_copy.add(f)
+
+    result = []
+    id = 0
+
+    json_data = config.filters_json_data["fileReplacements"]
+    dirname = json_data["directory"]
+    for directive in json_data["replacements"]:
+        input_file = LocalFile(dirname, directive["src"])
+        output_file = InFile(directive["dest"])
+        result += [
+            CopyRequest(
+                name = "input_copy_%d" % id,
+                input_file = input_file,
+                output_file = output_file
+            )
+        ]
+        files_to_copy.remove(output_file)
+        id += 1
+
+    for f in files_to_copy:
+        result += [
+            CopyRequest(
+                name = "input_copy_%d" % id,
+                input_file = SrcFile(f.filename),
+                output_file = f
+            )
+        ]
+        id += 1
+
+    result += requests
+    return result
+
+
 def main():
     args = flag_parser.parse_args()
     config = Config(args)
 
     if args.mode == "gnumake":
         makefile_vars = {
+            "SRC_DIR": "$(srcdir)",
             "IN_DIR": "$(srcdir)",
             "INDEX_NAME": "res_index"
         }
@@ -177,13 +218,14 @@ def main():
             key: "$(%s)" % key
             for key in list(makefile_vars.keys()) + makefile_env
         }
-        common["GLOB_DIR"] = args.in_dir
+        common["GLOB_DIR"] = args.src_dir
     else:
         common = {
             # GLOB_DIR is used now, whereas IN_DIR is used during execution phase.
             # There is no useful distinction in unix-exec or windows-exec mode.
-            "GLOB_DIR": args.in_dir,
-            "IN_DIR": args.in_dir,
+            "GLOB_DIR": args.src_dir,
+            "SRC_DIR": args.src_dir,
+            "IN_DIR": args.src_dir,
             "OUT_DIR": args.out_dir,
             "TMP_DIR": args.tmp_dir,
             "INDEX_NAME": "res_index",
@@ -192,17 +234,25 @@ def main():
         }
 
     def glob(pattern):
-        result_paths = pyglob.glob("{IN_DIR}/{PATTERN}".format(
-            IN_DIR = args.in_dir,
+        result_paths = pyglob.glob("{GLOB_DIR}/{PATTERN}".format(
+            GLOB_DIR = args.src_dir,
             PATTERN = pattern
         ))
         # For the purposes of buildtool, force Unix-style directory separators.
-        return [v.replace("\\", "/")[len(args.in_dir)+1:] for v in sorted(result_paths)]
+        return [v.replace("\\", "/")[len(args.src_dir)+1:] for v in sorted(result_paths)]
 
     requests = BUILDRULES.generate(config, glob, common)
     requests = filtration.apply_filters(requests, config)
     requests = utils.flatten_requests(requests, config, common)
 
+    if "fileReplacements" in config.filters_json_data:
+        tmp_in_dir = "{TMP_DIR}/in".format(**common)
+        if makefile_vars:
+            makefile_vars["IN_DIR"] = tmp_in_dir
+        else:
+            common["IN_DIR"] = tmp_in_dir
+        requests = add_copy_input_requests(requests, config, common)
+
     build_dirs = utils.compute_directories(requests)
 
     if args.mode == "gnumake":

diff --git a/icu4c/source/data/buildtool/filtration_schema.json b/icu4c/source/data/buildtool/filtration_schema.json
index 48e674e640a5f95f4814177344e1f6191fc4a388..f305fdc0d82d4cc62c4c343e614eefa1b14cab6b 100644 (file)
--- a/icu4c/source/data/buildtool/filtration_schema.json
+++ b/icu4c/source/data/buildtool/filtration_schema.json
@@ -33,6 +33,26 @@
                 "required": ["categories", "rules"],
                 "additionalProperties": false
             }
+        },
+        "fileReplacements": {
+            "type": "object",
+            "properties": {
+                "directory": { "type": "string" },
+                "replacements": {
+                    "type": "array",
+                    "items": {
+                        "type": "object",
+                        "properties": {
+                            "src": { "type": "string" },
+                            "dest": { "type": "string" }
+                        },
+                        "additionalProperties": false,
+                        "required": ["src", "dest"]
+                    }
+                }
+            },
+            "additionalProperties": false,
+            "required": ["directory", "replacements"]
         }
     },
     "additionalProperties": false,

diff --git a/icu4c/source/data/buildtool/request_types.py b/icu4c/source/data/buildtool/request_types.py
index 1890dd3c4ac8b9b5204dfdeedd78d98361760ddc..d5230c4132b0d2f5a9ad868e44df28178ace5018 100644 (file)
--- a/icu4c/source/data/buildtool/request_types.py
+++ b/icu4c/source/data/buildtool/request_types.py
@@ -104,6 +104,9 @@ class AbstractExecutionRequest(AbstractRequest):
         if not self.dep_targets:
             return
         for dep_target in self.dep_targets:
+            if isinstance(dep_target, InFile):
+                self.dep_files.append(dep_target)
+                continue
             for request in all_requests:
                 if request.name == dep_target.name:
                     self.dep_files += request.all_output_files()

diff --git a/icu4c/source/data/buildtool/utils.py b/icu4c/source/data/buildtool/utils.py
index d072428e90faf50ef3c7b180600197e511d7fadc..7fef140d3253f81c1164d9b1ef0b8ec39f15fcc7 100644 (file)
--- a/icu4c/source/data/buildtool/utils.py
+++ b/icu4c/source/data/buildtool/utils.py
@@ -11,6 +11,10 @@ from . import *
 
 
 def dir_for(file):
+    if isinstance(file, LocalFile):
+        return file.dirname
+    if isinstance(file, SrcFile):
+        return "{SRC_DIR}"
     if isinstance(file, InFile):
         return "{IN_DIR}"
     if isinstance(file, TmpFile):

diff --git a/icu4c/source/data/makedata.mak b/icu4c/source/data/makedata.mak
index 06fcc7267a7c19fb2a4e2a493204844756c40da4..c78ac6b6abd55183113fae3c6db24d3e68089286 100644 (file)
--- a/icu4c/source/data/makedata.mak
+++ b/icu4c/source/data/makedata.mak
@@ -235,7 +235,7 @@ $(COREDATA_TS):
        @cd "$(ICUSRCDATA)"
        py -3 -m buildtool \
                --mode windows-exec \
-               --in_dir "$(ICUSRCDATA)" \
+               --src_dir "$(ICUSRCDATA)" \
                --tool_dir "$(ICUTOOLS)" \
                --tool_cfg "$(CFG)" \
                --out_dir "$(ICUBLD_PKG)" \

diff --git a/icu4c/source/test/testdata/testdata.mak b/icu4c/source/test/testdata/testdata.mak
index 9d799f007440170f96cc89b67626db6a97b91cfb..a74ba08f329bdd4fa35c7dd813453c4ca4887e56 100644 (file)
--- a/icu4c/source/test/testdata/testdata.mak
+++ b/icu4c/source/test/testdata/testdata.mak
@@ -39,7 +39,7 @@ CREATE_DIRS :
                --mode windows-exec \
                --tool_dir "$(ICUTOOLS)" \
                --tool_cfg "$(CFG)" \
-               --in_dir "$(TESTDATA)" \
+               --src_dir "$(TESTDATA)" \
                --tmp_dir "$(TESTDATATMP)" \
                --out_dir "$(TESTDATABLD)"
        "$(ICUPBIN)\pkgdata" -f -v -m common -c -p"$(TESTPKG)" -d "$(TESTDATAOUT)" -T "$(TESTDATABLD)" -s "$(TESTDATABLD)" $(TESTDATATMP)\testdata.lst